Pre-compiled end-to-end gpu driver validation

Signed-off-by: shiva kumar <shivaku@nvidia.com>

.github/workflows/ci.yaml

@@ -20,7 +20,21 @@ on:
     types:
       - completed
     branches:
-      - main
+      - e2etestdriver
+
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      # - main
+      # - release-*
+      - e2etestdriver
+  push:
+    branches:
+      # - main
+      # - release-*
+      - e2etestdriver
 
 jobs:
   e2e-tests-nvidiadriver:
@@ -52,25 +66,26 @@ jobs:
       run: |
         echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV
         echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
+        echo "${{ secrets.AWS_SSH_KEY }}" > ${{ github.workspace }}/key.pem && chmod 400 ${{ github.workspace }}/key.pem
         echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
         DRIVER_VERSIONS=$(grep '^DRIVER_VERSIONS ?=' versions.mk | awk -F' ?= ' '{print $2}')
         echo "DRIVER_VERSIONS=$DRIVER_VERSIONS" >> $GITHUB_ENV
         
     - name: Validate gpu driver
       env:
         TEST_CASE: "./tests/cases/nvidia-driver.sh"
+        USE_PRECOMPILED: "0"
       run: |
-        echo "${{ secrets.AWS_SSH_KEY }}" > ${private_key} && chmod 400 ${private_key}
         rc=0
         for driver_version in ${DRIVER_VERSIONS}; do
           echo "Running e2e for DRIVER_VERSION=$driver_version"
-          ./tests/ci-run-e2e.sh ${TEST_CASE} ${COMMIT_SHORT_SHA}-${driver_version} || status=$?
+          status=0
+          ./tests/ci-run-e2e.sh ${TEST_CASE} ${COMMIT_SHORT_SHA}-${driver_version} ${USE_PRECOMPILED} || status=$?
           if [ $status -ne 0 ]; then
             echo "e2e validation failed for driver version $driver_version with status $status"
             rc=$status
           fi
         done
-        source ./tests/scripts/.definitions.sh
         ./tests/scripts/pull.sh /tmp/logs logs
         exit $rc
 
@@ -80,4 +95,39 @@ jobs:
       with:
         name: nvidiadriver-e2e-test-logs
         path: ./logs/
-        retention-days: 15  
+        retention-days: 15
+
+    - name: Precompiled e2e test- upgrade kernel and Validate gpu driver
+      env:
+        TEST_CASE_KERNEL_UPGRADE: "./tests/cases/nvidia-kernel-upgrade.sh"
+        TEST_CASE: "./tests/cases/nvidia-driver.sh"
+        USE_PRECOMPILED: "1"
+      run: |
+        rc=0
+        for driver_version in ${DRIVER_VERSIONS}; do
+          echo "Running e2e for DRIVER_VERSION=$driver_version"
+          status=0
+          ./tests/ci-run-e2e.sh ${TEST_CASE_KERNEL_UPGRADE} ${driver_version} ${USE_PRECOMPILED} || status=$?
+          if [ $status -ne 0 ]; then
+            echo "Kernel upgrade failed"
+            rc=$status
+          else
+            DRIVER_BRANCH=$(echo "${driver_version}" | cut -d '.' -f 1)
+            DRIVER_VERSION="${DRIVER_BRANCH}"
+            ./tests/ci-run-e2e.sh ${TEST_CASE} ${DRIVER_VERSION} ${USE_PRECOMPILED} || status=$?
+            if [ $status -ne 0 ]; then
+              echo "e2e validation failed for driver version $DRIVER_VERSION with status $status"
+              rc=$status
+            fi
+          fi
+        done
+        ./tests/scripts/pull.sh /tmp/logs logs
+        exit $rc
+
+    - name: Archive test logs
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: nvidiadriver-Precompiled-e2e-test-logs
+        path: ./logs/
+        retention-days: 15

.github/workflows/image.yaml

@@ -21,12 +21,14 @@ on:
       - opened
       - synchronize
     branches:
-      - main
-      - release-*
+      # - main
+      # - release-*
+      - e2etestdriver
   push:
     branches:
-      - main
-      - release-*
+      # - main
+      # - release-*
+      - e2etestdriver
 
 jobs:
   image:

tests/cases/nvidia-kernel-upgrade.sh

@@ -0,0 +1,8 @@
+#! /bin/bash
+# This test case runs the operator installation / test case with the default options.
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+source "${SCRIPTS_DIR}"/.definitions.sh
+
+# Run an end-to-end test cycle
+"${SCRIPTS_DIR}"/nvidia-kernel-upgrade-aws.sh

tests/ci-run-e2e.sh

@@ -2,14 +2,14 @@
 
 set -xe
 
-if [[ $# -ne 2 ]]; then
-	echo "TEST_CASE TARGET_DRIVER_VERSION are required"
+if [[ $# -ne 3 ]]; then
+	echo "TEST_CASE TARGET_DRIVER_VERSION USE_PRECOMPILED are required"
 	exit 1
 fi
 
 export TEST_CASE=${1}
 export TARGET_DRIVER_VERSION=${2}
-
+export USE_PRECOMPILED=${3}
 
 TEST_DIR="$(pwd)/tests"
 

tests/local.sh

@@ -12,6 +12,10 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/scripts && pwd )"
 source ${SCRIPT_DIR}/.definitions.sh
 source ${SCRIPT_DIR}/.local.sh
 
+if [ "${USE_PRECOMPILED}" == "1" ]; then
+    remote_retry
+fi
+
 # Sync the project folder to the remote
 ${SCRIPT_DIR}/push.sh
 
@@ -24,4 +28,5 @@ remote SKIP_PREREQUISITES="${SKIP_PREREQUISITES}" ./tests/scripts/prerequisites.
 remote \
     PROJECT="${PROJECT}" \
         TARGET_DRIVER_VERSION="${TARGET_DRIVER_VERSION}" \
+        USE_PRECOMPILED="${USE_PRECOMPILED}" \
         ${TEST_CASE}

tests/scripts/.definitions.sh

@@ -24,3 +24,8 @@ CASES_DIR="$( cd "${TEST_DIR}/cases" && pwd )"
 : ${POD_STATUS_TIME_OUT:="2m"}
 
 : ${LOG_DIR:="/tmp/logs"}
+
+: ${USE_PRECOMPILED:="0"}
+: ${SYSTEM_ONLINE_CHECK_TIMEOUT:="900"}
+
+: ${BASE_TARGET:="jammy"}

tests/scripts/.local.sh

@@ -3,3 +3,7 @@
 function remote() {
     ${SCRIPT_DIR}/remote.sh "cd ${PROJECT} && "$@""
 }
+
+function remote_retry() {
+    ${SCRIPT_DIR}/remote_retry.sh
+}

tests/scripts/.rsync-includes

@@ -0,0 +1,2 @@
+tests/
+tests/***

tests/scripts/end-to-end-nvidia-driver.sh

@@ -11,7 +11,7 @@ ${SCRIPT_DIR}/install-operator.sh
 
 "${SCRIPT_DIR}"/verify-operator.sh
 
-echo "--------------Verification completed for GPU Operator, uninstalling the operator--------------"
+echo "--------------Verification completed for GPU Operator, uninstalling the GPU operator--------------"
 
 ${SCRIPT_DIR}/uninstall-operator.sh ${TEST_NAMESPACE} "gpu-operator"
 

tests/scripts/findkernelversion.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [[ "${SKIP_INSTALL}" == "true" ]]; then
+    echo "Skipping install: SKIP_INSTALL=${SKIP_INSTALL}"
+    exit 0
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "${SCRIPT_DIR}"/.definitions.sh
+
+export REGCTL_VERSION=v0.4.7
+mkdir -p bin
+curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
+chmod a+x bin/regctl
+export PATH=$(pwd)/bin:${PATH}
+DRIVER_BRANCH=$(echo "${TARGET_DRIVER_VERSION}" | cut -d '.' -f 1)
+KERNEL_FLAVOR=$(uname -r | awk -F'-' '{print $3}')
+regctl image get-file ghcr.io/nvidia/driver:base-${BASE_TARGET}-${KERNEL_FLAVOR}-${DRIVER_BRANCH} /var/kernel_version.txt ${LOG_DIR}/kernel_version.txt || true

tests/scripts/install-operator.sh

@@ -5,9 +5,17 @@ if [[ "${SKIP_INSTALL}" == "true" ]]; then
     exit 0
 fi
 
+echo "Checking current kernel version..."
+CURRENT_KERNEL=$(uname -r)
+echo "Current kernel version: $CURRENT_KERNEL"
+
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 source ${SCRIPT_DIR}/.definitions.sh
 
+if [ "${USE_PRECOMPILED}" == "1" ]; then
+	OPERATOR_OPTIONS="${OPERATOR_OPTIONS} --set driver.usePrecompiled=true"
+fi
+
 OPERATOR_OPTIONS="${OPERATOR_OPTIONS} --set driver.repository=${PRIVATE_REGISTRY}/nvidia --set driver.version=${TARGET_DRIVER_VERSION}"
 
 # add helm driver repo 

tests/scripts/nvidia-kernel-upgrade-aws.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+if [[ "${SKIP_INSTALL}" == "true" ]]; then
+    echo "Skipping install: SKIP_INSTALL=${SKIP_INSTALL}"
+    exit 0
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "${SCRIPT_DIR}"/.definitions.sh
+
+# finding kernel version
+${SCRIPT_DIR}/findkernelversion.sh
+source "${LOG_DIR}"/kernel_version.txt
+
+echo "Checking current kernel version..."
+CURRENT_KERNEL=$(uname -r)
+echo "Current kernel version: $CURRENT_KERNEL"
+
+if [ "${CURRENT_KERNEL}" != ${KERNEL_VERSION} ]; then
+  echo ""
+  echo ""
+  echo "--------------Starting the Precompiled kernel version ${KERNEL_VERSION} upgrade--------------"
+
+  sudo apt-get update -y
+  sudo apt-get install --allow-downgrades linux-image-${KERNEL_VERSION} -y
+  if [ $? -ne 0 ]; then
+    echo "Kernel upgrade failed."
+    exit 1
+  fi
+
+  echo "update grub ..."
+  sudo sed -i "s/^GRUB_DEFAULT=.*/GRUB_DEFAULT=\"Advanced options for Ubuntu>Ubuntu, with Linux ${KERNEL_VERSION}\"/" /etc/default/grub
+  sudo cat /etc/default/grub | grep "GRUB_DEFAULT"
+  sudo update-grub
+
+  echo "Rebooting ..."
+  # Run the reboot command with nohup to avoid abrupt SSH closure issues
+  nohup sudo reboot &
+
+  echo "--------------Installation of kernel completed --------------"
+else
+  echo "--------------Installation is not required, kernel is already on ${KERNEL_VERSION} --------------"
+fi
+
+# Exit with a success code since the reboot command was issued successfully
+exit 0

tests/scripts/remote_retry.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source ${SCRIPT_DIR}/.definitions.sh
+source ${SCRIPT_DIR}/.local.sh
+
+try_ssh_connection() {
+    status=0
+    ssh -o ConnectTimeout=20 -i ${private_key} ${instance_hostname} "exit" || status=$?
+    return $status
+}
+
+echo "Waiting for aws system to come back online..."
+START_TIME=$(date +%s)
+while true; do
+    sleep 60
+    try_ssh_connection
+    if [ $? -eq 0 ]; then
+        echo "Successfully connected to aws system after reboot."
+        break;
+    fi
+    ELAPSED_TIME=$(($(date +%s) - START_TIME))
+    if [ "$ELAPSED_TIME" -ge "$SYSTEM_ONLINE_CHECK_TIMEOUT" ]; then
+        echo "Failed to connect to aws within ${SYSTEM_ONLINE_CHECK_TIMEOUT} minutes after reboot."
+        exit 1
+    fi
+    echo "ssh retry again..."
+done

tests/scripts/sync.sh

@@ -12,6 +12,7 @@ source ${SCRIPT_DIR}/.local.sh
 
 rsync -e "ssh -i ${private_key} -o StrictHostKeyChecking=no" \
     -avz --delete \
-    --exclude-from="${SCRIPT_DIR}/.rsync-excludes" \
+    --include-from="${SCRIPT_DIR}/.rsync-includes" \
+    --exclude='*' \
         ${@}