integration_tests.yml

name: Nightly integration tests

concurrency:
  # only one integration tests workflow to be run at a time, since it involves pushing/cleaning up a test image (same tag).
  group: ${{ github.workflow }}${{ github.event.workflow_run.name }}
  cancel-in-progress: false

# Run on request and every day at 3 AM UTC
on:
  workflow_dispatch:
    inputs:
      target:
        description: 'Target (choose nightly to run like nightly tests)'
        required: true
        default: 'nightly'
        type: choice
        options:
          - nightly
          - anyon
          - ionq
          - iqm
          - oqc
          - quantinuum
          - nvqc
          - orca
          - fermioniq
      single_test_name:
        type: string
        required: false
        description: 'Single test (e.g., targettests/quantinuum/load_value.cpp). Runs default tests if left blank'
      target_machine:
        type: string
        required: false
        description: 'Target machine (e.g., H1-1E).'
      cudaq_test_image:
        type: string
        required: false
        default: '' # picked up from repo variable if not provided
        description: 'CUDA Quantum image to run the tests in. Default to the latest CUDA Quantum nightly image'
      commit_sha:
        type: string
        required: false
        description: 'Commit SHA to pull the code (examples/tests) for testing. Default to the commit associated with the CUDA Quantum docker image if left blank'
      cudaq_nvqc_deploy_image:
        type: string
        required: false
        default: '' # same as cudaq_test_image if not provided
        description: 'CUDA Quantum image to use for NVQC deployment to NVCF. Default to the latest CUDA Quantum nightly image'
      workflow_id:
        type: string
        required: false
        description: 'Workflow Id to retrieve the Python wheel for testing. Default to the wheels produced by the Publishing workflow associated with the latest nightly CUDA Quantum Docker image if left blank'
      python_version:
        type: choice
        required: true
        description: 'Python version to run wheel test'
        options:
        - '3.10'
        - '3.11'
        - '3.12'

  schedule:
    - cron: 0 3 * * *

env:
  # NGC nv-quantum organization: pnyjrcojiblh
  NGC_QUANTUM_ORG: pnyjrcojiblh
  NGC_QUANTUM_TEAM: cuda-quantum
  NVQC_FUNCTION_ID: 3bfa0342-7d2a-4f1b-8e81-b6608d28ca7d
  # <Backend>:<GPU Type>:<Instance Type>:<Min Instances>:<Max Instances>
  NGC_NVQC_DEPLOYMENT_SPEC: GFN:L40S:gl40s_1.br25_2xlarge:1:1
  python_version: '3.10'

jobs:
  # We need this job purely to choose the container image values because the
  # `env` context is unavailable outside of "steps" contexts.
  setup:
    name: Set variables
    runs-on: ubuntu-latest
    outputs:
      cudaq_test_image: ${{ steps.vars.outputs.cudaq_test_image }}
      cudaq_nvqc_deploy_image: ${{ steps.vars.outputs.cudaq_nvqc_deploy_image }}
    steps:
      - name: Set variables
        id: vars
        run: |
          echo "cudaq_test_image=${{ inputs.cudaq_test_image || vars.cudaq_test_image }}" >> $GITHUB_OUTPUT
          echo "cudaq_nvqc_deploy_image=${{ inputs.cudaq_nvqc_deploy_image || vars.cudaq_test_image }}" >> $GITHUB_OUTPUT

  metadata:
    name: Retrieve commit info
    runs-on: ubuntu-latest
    needs: setup
    container:
      image: ${{ needs.setup.outputs.cudaq_test_image }}
      options: --user root
    outputs:
      cudaq_commit: ${{ steps.commit-sha.outputs.sha }}
    steps:
      - name: Get commit SHA
        id: commit-sha
        run: |
          if [ -n "${{ inputs.commit_sha }}" ]; then
            echo "sha=${{ inputs.commit_sha }}" >> $GITHUB_OUTPUT
          else
            echo "sha=$(cat $CUDA_QUANTUM_PATH/build_info.txt | grep -o 'source-sha: \S*' | cut -d ' ' -f 2)" >> $GITHUB_OUTPUT
          fi

  build_nvqc_image:
    name: Build NVQC deployment image
    runs-on: ubuntu-latest
    if: (inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
    needs: [setup, metadata]
    permissions:
      contents: read
      packages: write

    environment: ghcr-deployment

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.metadata.outputs.cudaq_commit }}
          fetch-depth: 1

      - name: Set up context for buildx
        run: |
          docker context create builder_context

      - name: Set up buildx runner
        uses: docker/setup-buildx-action@v3
        with:
          endpoint: builder_context

      - name: Login to NGC container registry
        uses: docker/login-action@v3
        with:
          registry: nvcr.io
          username: '$oauthtoken'
          password: ${{ secrets.NGC_CREDENTIALS }}

      # Log in to GHCR (in case the base image is a local one)
      - name: Log in to the GitHub container registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}

      - name: Build NVQC image
        id: docker_build
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./docker/release/cudaq.nvqc.Dockerfile
          build-args: |
            base_image=${{ needs.setup.outputs.cudaq_nvqc_deploy_image }}
          tags: nvcr.io/${{ env.NGC_QUANTUM_ORG }}/${{ env.NGC_QUANTUM_TEAM }}/${{ vars.packages_prefix }}cuda-quantum:nightly
          platforms: linux/amd64
          provenance: false
          push: true

  deploy_nvqc_test_function:
    name: Deploy NVQC function
    runs-on: ubuntu-latest
    needs: [metadata, build_nvqc_image]
    if: (inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
    permissions:
      contents: read

    # Must have environment protection
    environment: ghcr-deployment

    outputs:
      nvqc_function_version_id: ${{ steps.deploy.outputs.nvqc_function_version_id }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.metadata.outputs.cudaq_commit }}
          fetch-depth: 1

      - name: Install NGC CLI
        uses: ./.github/actions/install-ngc-cli
        with:
          version: 3.38.0
          checksum: 427c67684d792b673b63882a6d0cbb8777815095c0f2f31559c1570a91187388

      - name: Deploy NVQC Function
        id: deploy
        env:
          NGC_CLI_API_KEY: ${{ secrets.NGC_CREDENTIALS }}
          NGC_CLI_ORG: ${{ env.NGC_QUANTUM_ORG }}
          NGC_CLI_TEAM: cuda-quantum
        # When a new REST version is introduced, NVQC_REST_PAYLOAD_VERSION needs to be updated in lockstep with the new nightly CUDA Quantum image. 
        # Otherwise, deployment of the test function will fail.
        run: |
          # We run with CUDAQ_SER_CODE_EXEC set. The final NVQC deployment may
          # or may not have this set, but since we run the client with
          # CUDAQ_CLIENT_REMOTE_CAPABILITY_OVERRIDE=1 (below), we need to run
          # the CI with CUDAQ_SER_CODE_EXEC=1. If we ever remove
          # CUDAQ_CLIENT_REMOTE_CAPABILITY_OVERRIDE=1 below, we can consider
          # removing CUDAQ_SER_CODE_EXEC=1.
          create_function_result=$(ngc-cli/ngc cloud-function function create \
            --container-image nvcr.io/${{ env.NGC_QUANTUM_ORG }}/${{ env.NGC_QUANTUM_TEAM }}/cuda-quantum:nightly \
            --container-environment-variable NUM_GPUS:1 \
            --container-environment-variable NVQC_REST_PAYLOAD_VERSION:1.1 \
            --container-environment-variable RUN_AS_NOBODY:1 \
            --container-environment-variable CUDAQ_SER_CODE_EXEC:1 \
            --api-body-format CUSTOM \
            --inference-port 3030 \
            --health-uri / \
            --inference-url /job \
            --name cudaq-nightly-integration-test \
            $NVQC_FUNCTION_ID)
          version_id=$(echo "$create_function_result" | grep 'Version: \S*' | head -1 | cut -d ':' -f 2 | tr -d ' ')
          echo "Create version Id: $version_id"
          echo "nvqc_function_version_id=$version_id" >> $GITHUB_OUTPUT
          # Deploy it
          ngc-cli/ngc cloud-function function deploy create --deployment-specification $NGC_NVQC_DEPLOYMENT_SPEC $NVQC_FUNCTION_ID:$version_id
          function_status=DEPLOYING
          while [ "$function_status" = "DEPLOYING" ]; do
            echo "Waiting for deploying NVQC function version $version_id ..."
            sleep 120
            function_info=$(ngc-cli/ngc cloud-function function info $NVQC_FUNCTION_ID:$version_id)
            function_status=$(echo "$function_info" | grep 'Status: \S*' | head -1 | cut -d ':' -f 2 | tr -d ' ')
          done
          if [ "$function_status" != "ACTIVE" ]; then
            echo "::error:: Failed to deploy NVQC Test Function"
            exit 1
          fi

  hardware_providers_integration_test:
    name: Integration test with hardware providers
    runs-on: ubuntu-latest
    needs: [setup, metadata]
    permissions:
      contents: read
      packages: read

    # Must have environment protection
    environment: backend-validation
    container:
      image: ${{ needs.setup.outputs.cudaq_test_image }}
      options: --user root

    steps:
      - name: Get code
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.metadata.outputs.cudaq_commit }}
          fetch-depth: 1

      - name: Get tests
        id: gettests
        run: |
          if [ -n "${{ inputs.single_test_name }}" ]; then
            if [ -e ${{ inputs.single_test_name }} ]; then
              echo "testlist=${{ inputs.single_test_name }}" >> $GITHUB_OUTPUT
            else
              # User's request test does not exit
              echo "::error::File ${{ inputs.single_test_name}} not found"
              exit 1
            fi
          else
            filelist="targettests/*/*.cpp"
            echo "testlist=$filelist" >> $GITHUB_OUTPUT
          fi
      
      - name: Setup anyon account
        # This step is currently bypassed during nightly runs due to
        # maintenance. Restore the if check to the original value when
        # maintenance is complete.
        #if: github.event_name == 'schedule' || inputs.target == 'nightly' || inputs.target == 'anyon'
        if: inputs.target == 'anyon'
        run: |
          curl -X POST --user "${{ secrets.ANYON_USERNAME }}:${{ secrets.ANYON_PASSWORD }}"  -H "Content-Type: application/json" https://api.anyon.cloud:5000/login > credentials.json
          id_token=`cat credentials.json | jq -r '."id_token"'`
          refresh_token=`cat credentials.json | jq -r '."refresh_token"'`
          echo "key: $id_token" > ~/.anyon_config
          echo "refresh: $refresh_token" >> ~/.anyon_config

      - name: QIR syntax check (Anyon)
        # This step is currently bypassed during nightly runs due to
        # maintenance. Restore the if check to the original value when
        # maintenance is complete.
        #if: github.event_name == 'schedule' || inputs.target == 'nightly' || inputs.target == 'anyon'
        if: inputs.target == 'anyon'
        run: |
          echo "### QIR syntax check (Anyon)" >> $GITHUB_STEP_SUMMARY
          export CUDAQ_LOG_LEVEL="info"
          set +e # Allow script to keep going through errors
          test_err_sum=0
          for filename in ${{ steps.gettests.outputs.testlist }}; do
            case $filename in
              targettests/anyon/*)
                [ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
                nvq++ -v $filename -DSYNTAX_CHECK --target anyon --anyon-machine telegraph-8q
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  ./a.out
                  test_status=$?
                  if [ $test_status -eq 0 ]; then
                    echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
                  else
                    echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                    test_err_sum=$((test_err_sum+1))
                  fi
                else
                  echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              ;;
            esac
          done
          set -e # Re-enable exit code error checking
          if [ ! $test_err_sum -eq 0 ]; then
            echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
            exit 1
          fi
        shell: bash

      - name: Setup quantinum account
        if: github.event_name == 'schedule' || inputs.target == 'nightly' || inputs.target == 'quantinuum'
        run: |
          curl -X POST -H "Content Type: application/json" -d '{ "email":"${{ secrets.BACKEND_LOGIN_EMAIL }}","password":"${{ secrets.QUANTINUUM_PASSWORD }}" }' https://qapi.quantinuum.com/v1/login > credentials.json
          id_token=`cat credentials.json | jq -r '."id-token"'`
          refresh_token=`cat credentials.json | jq -r '."refresh-token"'`
          echo "key: $id_token" > ~/.quantinuum_config
          echo "refresh: $refresh_token" >> ~/.quantinuum_config

      - name: QIR syntax check (Quantinuum)
        if: github.event_name == 'schedule' || inputs.target == 'nightly'
        run: |
          echo "### QIR syntax check (Quantinuum)" >> $GITHUB_STEP_SUMMARY
          export CUDAQ_LOG_LEVEL="info"
          set +e # Allow script to keep going through errors
          test_err_sum=0
          for filename in ${{ steps.gettests.outputs.testlist }}; do
            case $filename in
              targettests/quantinuum/*)
                [ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
                nvq++ -v $filename -DSYNTAX_CHECK --target quantinuum --quantinuum-machine H1-1SC
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  ./a.out
                  test_status=$?
                  if [ $test_status -eq 0 ]; then
                    echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
                  else
                    echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                    test_err_sum=$((test_err_sum+1))
                  fi
                else
                  echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              ;;
            esac
          done
          set -e # Re-enable exit code error checking
          if [ ! $test_err_sum -eq 0 ]; then
            echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
            exit 1
          fi
        shell: bash

      - name: Submit to IonQ Simulator
        if: (success() || failure()) && (inputs.target == 'ionq' || github.event_name == 'schedule' || inputs.target == 'nightly')
        run: |
          echo "### Submit to IonQ Simulator" >> $GITHUB_STEP_SUMMARY
          export IONQ_API_KEY='${{ secrets.IONQ_API_KEY }}'
          # TODO: remove this flag once https://github.com/NVIDIA/cuda-quantum/issues/512 is addressed.
          export CUDAQ_LOG_LEVEL="info"
          set +e # Allow script to keep going through errors
          test_err_sum=0
          for filename in ${{ steps.gettests.outputs.testlist }}; do
            case $filename in
              targettests/ionq/*)
                [ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
                nvq++ -v $filename --target ionq
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  ./a.out
                  test_status=$?
                  if [ $test_status -eq 0 ]; then
                    echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
                  else
                    echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                    test_err_sum=$((test_err_sum+1))
                  fi
                else
                  echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              ;;
            esac
          done
          set -e # Re-enable exit code error checking
          if [ ! $test_err_sum -eq 0 ]; then
            echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
            exit 1
          fi
        shell: bash

      - name: Submit to IQM Demo server
        if: (success() || failure()) && (inputs.target == 'iqm' || github.event_name == 'schedule' || inputs.target == 'nightly')
        run: |
          # Must install iqm-cortex-cli to authenticate
          pip install iqm-cortex-cli
          echo "### Submit to IQM Demo server" >> $GITHUB_STEP_SUMMARY
          # IQM demo server info is from: https://demo.qc.iqm.fi/cocos/info/
          cortex init --config-file ${HOME}/.config/iqm-cortex-cli/config.json --tokens-file ${HOME}/.cache/iqm-cortex-cli/tokens.json --auth-server-url https://demo.qc.iqm.fi/auth --client-id iqm_client --realm cortex --username '${{ secrets.IQM_USER }}'
          cortex auth login --username '${{ secrets.IQM_USER }}' --password '${{ secrets.IQM_PASSWORD }}'
          echo ":white_check_mark: Successfully installed iqm-cortex-cli and logged in" >> $GITHUB_STEP_SUMMARY
          # Use the demo machine, which is Adonis architecture
          export IQM_SERVER_URL="https://demo.qc.iqm.fi/cocos"
          export CUDAQ_LOG_LEVEL="info"
          set +e # Allow script to keep going through errors
          test_err_sum=0
          for filename in ${{ steps.gettests.outputs.testlist }}; do
            [ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
            # Only the following tests are currently supported on IQM
            case $filename in
              targettests/iqm/*)
                nvq++ -DSYNTAX_CHECK --target iqm --iqm-machine Adonis $filename
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  ./a.out
                  test_status=$?
                  if [ $test_status -eq 0 ]; then
                    echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
                  else
                    echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                    test_err_sum=$((test_err_sum+1))
                  fi
                else
                  echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              ;;
            esac
          done
          set -e # Re-enable exit code error checking
          cortex auth logout -f
          echo ":white_check_mark: Successfully logged out of IQM" >> $GITHUB_STEP_SUMMARY
          if [ ! $test_err_sum -eq 0 ]; then
            echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
            exit 1
          fi
        shell: bash

      - name: Submit to OQC Sandbox server
        # This step is currently bypassed during nightly runs due to
        # maintenance. Restore the if check to the original value when
        # maintenance is complete.
        # if: (success() || failure()) && (inputs.target == 'oqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
        if: (success() || failure()) && (inputs.target == 'oqc')
        run: |
          echo "### Submit to OQC sandbox server" >> $GITHUB_STEP_SUMMARY
          export CUDAQ_LOG_LEVEL="info"
          export OQC_EMAIL='${{ secrets.BACKEND_LOGIN_EMAIL }}'
          export OQC_PASSWORD='${{ secrets.OQC_PASSWORD }}'
          set +e # Allow script to keep going through errors
          test_err_sum=0
          for filename in ${{ steps.gettests.outputs.testlist }}; do
            [ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
            # Only the following tests are currently supported on OQC
            case $filename in
              targettests/oqc/*)
                nvq++ -DSYNTAX_CHECK --target oqc $filename
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  ./a.out
                  test_status=$?
                  if [ $test_status -eq 0 ]; then
                    echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
                  else
                    echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                    test_err_sum=$((test_err_sum+1))
                  fi
                else
                  echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              ;;
            esac
          done
          set -e # Re-enable exit code error checking
          if [ ! $test_err_sum -eq 0 ]; then
            echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
            exit 1
          fi
        shell: bash

      - name: Submit to ORCA test server
        if: (success() || failure()) && (inputs.target == 'orca' || github.event_name == 'schedule' || inputs.target == 'nightly')
        run: |
          echo "### Submit to ORCA server" >> $GITHUB_STEP_SUMMARY
          export ORCA_ACCESS_URL='${{ secrets.ORCA_ACCESS_URL }}'
          set +e # Allow script to keep going through errors
          test_err_sum=0
          cpp_tests="docs/sphinx/targets/cpp/orca.cpp  docs/sphinx/targets/cpp/orca_mqpu.cpp"
          for filename in $cpp_tests; do
            [ -e "$filename" ] || echo "::error::Couldn't find file ($filename)"
            nvq++ --target orca --orca-url $ORCA_ACCESS_URL $filename
            test_status=$?
            if [ $test_status -eq 0 ]; then
              ./a.out
              test_status=$?
              if [ $test_status -eq 0 ]; then
                echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
              else
                echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                test_err_sum=$((test_err_sum+1))
              fi
            else
              echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
              test_err_sum=$((test_err_sum+1))
            fi
          done
          python_tests="docs/sphinx/targets/python/orca.py  docs/sphinx/targets/python/orca_mqpu.py"
          for filename in $python_tests; do
            [ -e "$filename" ] || echo "::error::Couldn't find file ($filename)"
            python3 $filename 1> /dev/null
            test_status=$?
            if [ $test_status -eq 0 ]; then
              echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
            else
              echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
              test_err_sum=$((test_err_sum+1))
            fi
          done
          set -e # Re-enable exit code error checking
          if [ ! $test_err_sum -eq 0 ]; then
            echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
            exit 1
          fi
        shell: bash

      - name: Submit to Fermioniq test server
        if: (success() || failure()) && (inputs.target == 'fermioniq' || github.event_name == 'schedule' || inputs.target == 'nightly')
        run: |
          echo "### Submit to Fermioniq server" >> $GITHUB_STEP_SUMMARY
          export FERMIONIQ_ACCESS_TOKEN_ID='${{ secrets.FERMIONIQ_ACCESS_TOKEN_ID }}'
          export FERMIONIQ_ACCESS_TOKEN_SECRET='${{ secrets.FERMIONIQ_ACCESS_TOKEN_SECRET }}'
          export FERMIONIQ_PROJECT_ID='c86e2cb8-d776-41ae-8240-d3d55e18b7c3'
          set +e # Allow script to keep going through errors
          test_err_sum=0
          cpp_tests="docs/sphinx/targets/cpp/fermioniq.cpp"
          for filename in $cpp_tests; do
            [ -e "$filename" ] || echo "::error::Couldn't find file ($filename)"
            nvq++ \
              --target fermioniq \
              --fermioniq-project-id $FERMIONIQ_PROJECT_ID \
              $filename
            test_status=$?
            if [ $test_status -eq 0 ]; then
              ./a.out 1> /dev/null
              test_status=$?
              if [ $test_status -eq 0 ]; then
                echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
              else
                echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                test_err_sum=$((test_err_sum+1))
              fi
            else
              echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
              test_err_sum=$((test_err_sum+1))
            fi
          done
          python_tests="docs/sphinx/targets/python/fermioniq.py docs/sphinx/targets/python/fermioniq_observables.py"
          for filename in $python_tests; do
            [ -e "$filename" ] || echo "::error::Couldn't find file ($filename)"
            python3 $filename 1> /dev/null
            test_status=$?
            if [ $test_status -eq 0 ]; then
              echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
            else
              echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
              test_err_sum=$((test_err_sum+1))
            fi
          done
          set -e # Re-enable exit code error checking
          if [ ! $test_err_sum -eq 0 ]; then
            echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
            exit 1
          fi
        shell: bash

      - name: Submit to ${{ inputs.target }}
        # The full set of tests used by this step is currently only supported on
        # Quantinuum.  The other supported tests are tested by the step above.
        # The main point of this special step is to run with a special
        # target_machine. It also doesn't use -DSYNTAX_CHECK, so you probably
        # don't want to run this on a simple Syntax Check machine.
        if: inputs.target == 'quantinuum' && github.event_name == 'workflow_dispatch'
        run: |
          if ${{inputs.target == 'ionq'}}; then
            export IONQ_API_KEY='${{ secrets.IONQ_API_KEY }}'
          fi
          for filename in ${{ steps.gettests.outputs.testlist }}; do
            [ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
            case $filename in
              targettests/quantinuum/*)
                nvq++ -v $filename --target ${{ inputs.target }} --${{ inputs.target }}-machine ${{ inputs.target_machine }}
                CUDAQ_LOG_LEVEL=info ./a.out
              ;;
            esac
          done
        shell: bash

  nvqc_integration_docker_test:
    name: NVQC integration test using Docker image
    runs-on: ubuntu-latest
    if: (inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
    needs: [setup, metadata, build_nvqc_image, deploy_nvqc_test_function]
    permissions:
      contents: read
      packages: read

    # Must have environment protection
    environment: ghcr-deployment
    container:
      image: ${{ needs.setup.outputs.cudaq_test_image }}
      options: --user root

    steps:
      - name: Get code
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.metadata.outputs.cudaq_commit }}
          fetch-depth: 1

      - name: Submit to NVQC
        run: |
          echo "### Submit to NVQC" >> $GITHUB_STEP_SUMMARY
          export NVQC_API_KEY="${{ secrets.NVQC_SERVICE_KEY }}"
          export NVQC_FUNCTION_ID="$NVQC_FUNCTION_ID"
          export NVQC_FUNCTION_VERSION_ID="${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}"
          # When overriding the NVQC_FUNCTION_ID to a function that doesn't
          # follow the production naming convenvtions, we need to set the
          # following environment variable to tell the client that the server
          # has all the remote capabilities.
          export CUDAQ_CLIENT_REMOTE_CAPABILITY_OVERRIDE=1
          set +e # Allow script to keep going through errors
          test_err_sum=0
          # Test all NVQPP execution tests
          for filename in `find targettests/execution/ -name '*.cpp'`; do
            echo "$filename"
            # Only run tests that require execution (not a syntax-only check)
            if grep -q "ifndef SYNTAX_CHECK" "$filename"; then
              nvq++ -v $filename --target nvqc
              test_status=$?
              if [ $test_status -eq 0 ]; then
                ./a.out
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
                else
                  echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              else
                echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
                test_err_sum=$((test_err_sum+1))
              fi
            fi
          done

          # Test all remote-sim tests
          for filename in `find targettests/Remote-Sim -name '*.cpp'`; do
            # unsupport_args and compile_errors are compile error tests
            # nested_vectors: Compiler fails to synthesize nested vector parameters (https://github.com/NVIDIA/cuda-quantum/issues/2001)
            # qvector_init_from_state, qvector_init_from_state_lazy, test_trotter: New argument synthesis is not executed for nvqc (https://github.com/NVIDIA/cuda-quantum/issues/2146)
            if [[ "$filename" != *"unsupport_args"* ]] && [[ "$filename" != *"compile_errors"* ]] && [[ "$filename" != *"nested_vectors"* ]] && [[ "$filename" != *"qvector_init_from_state"* ]] && [[ "$filename" != *"qvector_init_from_state_lazy"* ]] && [[ "$filename" != *"test_trotter"* ]]; then
              echo "$filename"
              nvqc_config=""
              # Look for a --remote-mqpu-auto-launch to determine the number of QPUs
              num_qpus=`cat $filename | grep -oP -m 1 '^//\s*RUN:\s*nvq++.+--remote-mqpu-auto-launch\s+\K\S+'`
              if [ -n "$num_qpus" ]; then
                echo "Intended to run on '$num_qpus' QPUs."
                nvqc_config="$nvqc_config --nvqc-nqpus $num_qpus"
              fi
              nvq++ -v $filename --target nvqc $nvqc_config
              test_status=$?
              if [ $test_status -eq 0 ]; then
                ./a.out
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
                else
                  echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              else
                  echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
              fi
            fi
          done

          # Test C++ examples with NVQC
          for filename in `find examples/cpp/ applications/cpp/ targets/cpp/ -name '*.cpp'`; do
            if [[ "$filename" == *"nvqc"* ]]; then
              echo "$filename"
              nvqc_config=""
              # Look for a --nvqc-backend flag to nvq++ in the comment block
              nvqc_backend=`sed -e '/^$/,$d' $filename | grep -oP -m 1 '^//\s*nvq++.+--nvqc-backend\s+\K\S+'`
              if [ -n "$nvqc_backend" ]; then
                echo "Intended for execution on '$nvqc_backend' backend."
                nvqc_config="$nvqc_config --nvqc-backend $nvqc_backend"
              fi
              # Look for a --nvqc-nqpus flag to nvq++ in the comment block
              num_qpus=`sed -e '/^$/,$d' $filename | grep -oP -m 1 '^//\s*nvq++.+--nvqc-nqpus\s+\K\S+'`
              if [ -n "$num_qpus" ]; then
                echo "Intended to run on '$num_qpus' QPUs."
                nvqc_config="$nvqc_config --nvqc-nqpus $num_qpus"
              fi
              nvq++ -v $filename --target nvqc $nvqc_config
              test_status=$?
              if [ $test_status -eq 0 ]; then
                ./a.out
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
                else
                  echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              else
                  echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
              fi
            fi
          done

          # Test NVQC Python examples + Python MLIR execution tests (not IR tests)
          python3 -m pip install pytest
          for ex in `find examples/python python/tests/mlir/target -name '*.py'`; do
            filename=$(basename -- "$ex")
            filename="${filename%.*}"
            echo "Testing $filename:"
            if [[ "$ex" == *"nvqc"* ]]; then
              # This is an NVQC example
              python3 $ex 1> /dev/null
              test_status=$?
              if [ $test_status -eq 0 ]; then
                echo ":white_check_mark: Successfully ran test: $ex" >> $GITHUB_STEP_SUMMARY
              else
                echo ":x: Test failed (failed to execute): $ex" >> $GITHUB_STEP_SUMMARY
                test_err_sum=$((test_err_sum+1))
              fi
            # building_kernels.py is disabled due to https://github.com/NVIDIA/cuda-quantum/issues/2299.
            elif [[ "$ex" != *"building_kernels"* ]]; then
              # Only run examples that are not target-specific (e.g., ionq, iqm)
              if ! grep -q "set_target" "$ex"; then
                # Use --target command line option to run these examples with nvqc
                python3 $ex --target nvqc 1> /dev/null
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  echo ":white_check_mark: Successfully ran test: $ex" >> $GITHUB_STEP_SUMMARY
                else
                  echo ":x: Test failed (failed to execute): $ex" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              fi
            fi
          done

          set -e # Re-enable exit code error checking
          if [ ! $test_err_sum -eq 0 ]; then
            echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
            exit 1
          fi
        shell: bash

  nvqc_integration_wheel_test:
    name: NVQC integration test using Python wheels
    runs-on: ubuntu-latest
    if: inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly'
    needs: [metadata, build_nvqc_image, deploy_nvqc_test_function]
    permissions:
      contents: read

    # Must have environment protection
    environment: ghcr-deployment

    steps:
      - name: Get code
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.metadata.outputs.cudaq_commit }}
          fetch-depth: 1

      - name: Install wheel
        id: install_wheel
        run: |
          python_version=${{ inputs.python_version || env.python_version }}
          workflow_id=${{ inputs.workflow_id }}
          # Helper to get the *valid* Publishing run Id for a commit hash
          # Notes: runs that have 'CUDA Quantum Python wheels' jobs skipped are not considered.
          function get_publishing_run_id {
            # Find all Publishing runs, we'll look into its jobs' status later
            if [[ -z "$1" ]]; then
              publishing_run_ids=$(gh run list --workflow Publishing --json databaseId --jq .[].databaseId)
            else
              publishing_run_ids=$(gh run list --commit $1 --workflow Publishing --json databaseId --jq .[].databaseId)
            fi
            for run_id in $publishing_run_ids ; do
                # Look into its jobs: if "CUDA Quantum Python wheels" matrix build was performed,
                # then we have multiple jobs, like "CUDA Quantum Python wheels (python_arm64....")
                cuda_wheel_build_jobs=$(gh run view $run_id --jq '.jobs.[] | select(.name | startswith("CUDA Quantum Python wheels (python_")).name' --json jobs)
                if [ ! -z "$cuda_wheel_build_jobs" ]; then
                  # This is a valid run that produces wheel artifacts
                  echo $run_id
                  break
                fi
            done
          }

          if [ -z "${workflow_id}" ]; then
            workflow_id=$(get_publishing_run_id ${{ needs.metadata.outputs.cudaq_commit }})
          fi
          if [ ! -z "$workflow_id" ]; then
            echo "Using artifacts from workflow id $workflow_id"
            # Allow error when trying to download wheel artifacts since they might be expired.
            set +e
            gh run download $workflow_id --name "x86_64-py$python_version-wheels"
            retVal=$?
            set -e
            if [ $retVal -ne 0 ]; then
              echo "Failed to download wheels artifact from Publishing workflow run Id $workflow_id. Perhaps the artifacts have been expired."
              # This is allowed since there might be a period where no Publishing workflow is run (e.g., no PR merged to main).
              echo "skipped=true" >> $GITHUB_OUTPUT
              exit 0
            fi
            python_version_filename=$(echo "${python_version//.}")
            # Install Python and the wheel
            apt-get update && apt-get install -y --no-install-recommends python$python_version python3-pip
            wheelfile=$(find . -name "cuda_quantum_cu11*cp$python_version_filename*x86_64.whl")
            python$python_version -m pip install $wheelfile
            echo "skipped=false" >> $GITHUB_OUTPUT
          else
            echo "Failed to retrieve Publishing workflow run Id for commit ${{ needs.metadata.outputs.cudaq_commit }}"
            exit 1
          fi
        env:
          GH_TOKEN: ${{ github.token }}

      - name: Test NVQC
        if: ${{ ! steps.install_wheel.skipped }}
        run: |
          echo "### Submit to NVQC from Python wheels" >> $GITHUB_STEP_SUMMARY
          python_version=${{ inputs.python_version || env.python_version }}
          export NVQC_API_KEY="${{ secrets.NVQC_SERVICE_KEY }}"
          export NVQC_FUNCTION_ID="$NVQC_FUNCTION_ID"
          export NVQC_FUNCTION_VERSION_ID="${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}"
          set +e # Allow script to keep going through errors
          python$python_version -m pip install pytest
          test_err_sum=0
          for ex in `find examples/python python/tests/mlir/target -name '*.py'`; do
            filename=$(basename -- "$ex")
            filename="${filename%.*}"
            echo "Testing $filename:"
            if [[ "$ex" == *"nvqc"* ]]; then
              python$python_version $ex 1> /dev/null
              test_status=$?
              if [ $test_status -eq 0 ]; then
                echo ":white_check_mark: Successfully ran test: $ex" >> $GITHUB_STEP_SUMMARY
              else
                echo ":x: Test failed (failed to execute): $ex" >> $GITHUB_STEP_SUMMARY
                test_err_sum=$((test_err_sum+1))
              fi
            # building_kernels.py is disabled due to https://github.com/NVIDIA/cuda-quantum/issues/2299.
            elif [[ "$ex" != *"building_kernels"* ]]; then
              # Only run examples that are not target-specific (e.g., ionq, iqm)
              if ! grep -q "set_target" "$ex"; then
                # Use --target command line option to run these examples with nvqc
                python$python_version $ex --target nvqc 1> /dev/null
                test_status=$?
                if [ $test_status -eq 0 ]; then
                  echo ":white_check_mark: Successfully ran test: $ex" >> $GITHUB_STEP_SUMMARY
                else
                  echo ":x: Test failed (failed to execute): $ex" >> $GITHUB_STEP_SUMMARY
                  test_err_sum=$((test_err_sum+1))
                fi
              fi
            fi
          done
          set -e # Re-enable exit code error checking
          if [ ! $test_err_sum -eq 0 ]; then
            echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
            exit 1
          fi

  cleanup_nvqc_resources:
    name: Cleanup NVQC resources
    runs-on: ubuntu-latest
    if: (success() || failure()) && (inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
    needs: [build_nvqc_image, deploy_nvqc_test_function, nvqc_integration_docker_test, nvqc_integration_wheel_test]
    permissions:
      contents: read

    # Must have environment protection
    environment: ghcr-deployment

    steps:
      - name: Get code
        uses: actions/checkout@v4

      - name: Install NGC CLI
        uses: ./.github/actions/install-ngc-cli
        with:
          version: 3.38.0
          checksum: 427c67684d792b673b63882a6d0cbb8777815095c0f2f31559c1570a91187388

      - name: Cleanup
        env:
          NGC_CLI_API_KEY: ${{ secrets.NGC_CREDENTIALS }}
          NGC_CLI_ORG: ${{ env.NGC_QUANTUM_ORG }}
          NGC_CLI_TEAM: cuda-quantum
        run: |
          echo "Version Id: ${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}"
          # Remove deployment (make it inactive)
          ngc-cli/ngc cloud-function function deploy remove $NVQC_FUNCTION_ID:${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}
          # Remove the function version
          ngc-cli/ngc cloud-function function remove $NVQC_FUNCTION_ID:${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}
          # Remove the docker image
          ngc-cli/ngc registry image remove -y nvcr.io/${{ env.NGC_QUANTUM_ORG }}/${{ env.NGC_QUANTUM_TEAM }}/cuda-quantum:nightly