diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh
index 0be9573253..9208d1d577 100755
--- a/parts/k8s/cloud-init/artifacts/cse_main.sh
+++ b/parts/k8s/cloud-init/artifacts/cse_main.sh
@@ -50,6 +50,12 @@ else
     REBOOTREQUIRED=false
 fi
 
+if [[ "$CONTAINER_RUNTIME" != "kata-containers" ]] && [[ "$CONTAINER_RUNTIME" != "containerd" ]]; then
+  cleanUpContainerd
+fi
+if [[ "${GPU_NODE}" != "true" ]]; then
+  cleanUpGPUDrivers
+fi
 
 VHD_LOGS_FILEPATH=/opt/azure/vhd-install.complete
 if [[ "${IS_VHD}" = true ]]; then
@@ -81,16 +87,12 @@ fi
 installNetworkPlugin
 if [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
     installContainerd
-else
-    cleanUpContainerd
 fi
 if [[ "${GPU_NODE}" = true ]]; then
     if $FULL_INSTALL_REQUIRED; then
         installGPUDrivers
     fi
     ensureGPUDrivers
-else
-    cleanUpGPUDrivers
 fi
 installKubeletAndKubectl
 if [[ $OS != $COREOS_OS_NAME ]]; then
diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go
index a11f554a54..af4cd9052b 100644
--- a/pkg/engine/templates_generated.go
+++ b/pkg/engine/templates_generated.go
@@ -13713,6 +13713,12 @@ else
     REBOOTREQUIRED=false
 fi
 
+if [[ "$CONTAINER_RUNTIME" != "kata-containers" ]] && [[ "$CONTAINER_RUNTIME" != "containerd" ]]; then
+  cleanUpContainerd
+fi
+if [[ "${GPU_NODE}" != "true" ]]; then
+  cleanUpGPUDrivers
+fi
 
 VHD_LOGS_FILEPATH=/opt/azure/vhd-install.complete
 if [[ "${IS_VHD}" = true ]]; then
@@ -13744,16 +13750,12 @@ fi
 installNetworkPlugin
 if [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
     installContainerd
-else
-    cleanUpContainerd
 fi
 if [[ "${GPU_NODE}" = true ]]; then
     if $FULL_INSTALL_REQUIRED; then
         installGPUDrivers
     fi
     ensureGPUDrivers
-else
-    cleanUpGPUDrivers
 fi
 installKubeletAndKubectl
 if [[ $OS != $COREOS_OS_NAME ]]; then