Opt in device plugin

e2e2/test/cases/neuron/main_test.go

@@ -19,8 +19,9 @@ import (
 )
 
 var (
- testenv env.Environment
- neuronTestImage *string
+ testenv env.Environment
+ neuronTestImage *string
+ installDevicePlugin *bool
 )
 
 var (
@@ -32,6 +33,7 @@ var (
 
 func TestMain(m *testing.M) {
  neuronTestImage = flag.String("neuronTestImage", "", "image for neuron single node test")
+ installDevicePlugin = flag.Bool("installDevicePlugin", true, "install nvidia device plugin")
  cfg, err := envconf.NewFromFlags()
  if err != nil {
  log.Fatalf("failed to initialize test environment: %v", err)
@@ -41,9 +43,10 @@ func TestMain(m *testing.M) {
  defer cancel()
  testenv = testenv.WithContext(ctx)
 
- manifests := [][]byte{
- neuronDevicePluginManifest,
- neuronDevicePlugiRbacManifest,
+ var manifests [][]byte
+
+ if *installDevicePlugin {
+ manifests = append(manifests, neuronDevicePluginManifest, neuronDevicePlugiRbacManifest)
  }
 
  testenv.Setup(
@@ -55,6 +58,9 @@ func TestMain(m *testing.M) {
  return ctx, nil
  },
  func(ctx context.Context, config *envconf.Config) (context.Context, error) {
+ if !*installDevicePlugin {
+ return ctx, nil
+ }
  ds := appsv1.DaemonSet{
  ObjectMeta: metav1.ObjectMeta{Name: "neuron-device-plugin-daemonset", Namespace: "kube-system"},
  }

e2e2/test/cases/nvidia/main_test.go

@@ -24,13 +24,14 @@ import (
 )
 
 var (
- testenv env.Environment
- nodeType *string
- efaEnabled *bool
- nvidiaTestImage *string
- nodeCount int
- gpuPerNode int
- efaPerNode int
+ testenv env.Environment
+ nodeType *string
+ installDevicePlugin *bool
+ efaEnabled *bool
+ nvidiaTestImage *string
+ nodeCount int
+ gpuPerNode int
+ efaPerNode int
 )
 
 var (
@@ -46,6 +47,7 @@ func TestMain(m *testing.M) {
  nodeType = flag.String("nodeType", "", "node type for the tests")
  nvidiaTestImage = flag.String("nvidiaTestImage", "", "nccl test image for nccl tests")
  efaEnabled = flag.Bool("efaEnabled", false, "enable efa tests")
+ installDevicePlugin = flag.Bool("installDevicePlugin", true, "install nvidia device plugin")
  cfg, err := envconf.NewFromFlags()
  if err != nil {
  log.Fatalf("failed to initialize test environment: %v", err)
@@ -57,9 +59,11 @@ func TestMain(m *testing.M) {
 
  // all NVIDIA tests require the device plugin and MPI operator
  manifests := [][]byte{
- nvidiaDevicePluginManifest,
  mpiOperatorManifest,
  }
+ if *installDevicePlugin {
+ manifests = append(manifests, nvidiaDevicePluginManifest)
+ }
 
  testenv.Setup(
  func(ctx context.Context, config *envconf.Config) (context.Context, error) {
@@ -81,6 +85,9 @@ func TestMain(m *testing.M) {
  return ctx, nil
  },
  func(ctx context.Context, config *envconf.Config) (context.Context, error) {
+ if !*installDevicePlugin {
+ return ctx, nil
+ }
  ds := appsv1.DaemonSet{
  ObjectMeta: metav1.ObjectMeta{Name: "nvidia-device-plugin-daemonset", Namespace: "kube-system"},
  }

kubetest2/internal/deployers/eksapi/nodegroup.go

@@ -157,7 +157,7 @@ func (m *NodegroupManager) createManagedNodegroup(infra *Infrastructure, cluster
 func (m *NodegroupManager) createUnmanagedNodegroup(infra *Infrastructure, cluster *Cluster, opts *deployerOptions) error {
  stackName := m.getUnmanagedNodegroupStackName()
  klog.Infof("creating unmanaged nodegroup stack...")
- userData, userDataIsMimePart, err := generateUserData(opts.UserDataFormat, opts.EFA, cluster)
+ userData, userDataIsMimePart, err := generateUserData(opts.UserDataFormat, cluster)
  if err != nil {
  return err
  }
@@ -258,7 +258,7 @@ func (m *NodegroupManager) createUnmanagedNodegroup(infra *Infrastructure, clust
 func (m *NodegroupManager) createUnmanagedNodegroupWithEFA(infra *Infrastructure, cluster *Cluster, opts *deployerOptions) error {
  stackName := m.getUnmanagedNodegroupStackName()
  klog.Infof("creating unmanaged nodegroup with EFA stack...")
- userData, userDataIsMimePart, err := generateUserData(opts.UserDataFormat, opts.EFA, cluster)
+ userData, userDataIsMimePart, err := generateUserData(opts.UserDataFormat, cluster)
  if err != nil {
  return err
  }

kubetest2/internal/deployers/eksapi/templates/templates.go

@@ -41,7 +41,6 @@ type UserDataTemplateData struct {
  CertificateAuthority string
  CIDR string
  APIServerEndpoint string
- EFAEnabled bool
 }
 
 var (

kubetest2/internal/deployers/eksapi/templates/userdata_bottlerocket.toml.template

@@ -5,6 +5,3 @@
 
 [settings.host-containers.admin]
 "enabled" = true
-
-[settings.efa]
-"enabled" = {{.EFAEnabled}}

kubetest2/internal/deployers/eksapi/userdata.go

@@ -8,7 +8,7 @@ import (
  "github.com/aws/aws-k8s-tester/kubetest2/internal/deployers/eksapi/templates"
 )
 
-func generateUserData(format string, efaEnabled bool, cluster *Cluster) (string, bool, error) {
+func generateUserData(format string, cluster *Cluster) (string, bool, error) {
  userDataIsMimePart := true
  var t *template.Template
  switch format {
@@ -29,7 +29,6 @@ func generateUserData(format string, efaEnabled bool, cluster *Cluster) (string,
  CertificateAuthority: cluster.certificateAuthorityData,
  CIDR: cluster.cidr,
  Name: cluster.name,
- EFAEnabled: efaEnabled,
  }); err != nil {
  return "", false, err
  }

kubetest2/internal/deployers/eksapi/userdata_test.go

@@ -42,40 +42,33 @@ const bottlerocketUserData = `[settings.kubernetes]
 
 [settings.host-containers.admin]
 "enabled" = true
-
-[settings.efa]
-"enabled" = true
 `
 
 func Test_generateUserData(t *testing.T) {
  cases := []struct {
  format string
- efa bool
  expected string
  expectedIsMimePart bool
  }{
  {
  format: "bootstrap.sh",
- efa: false,
  expected: bootstrapShUserData,
  expectedIsMimePart: true,
  },
  {
  format: "nodeadm",
- efa: false,
  expected: nodeadmUserData,
  expectedIsMimePart: true,
  },
  {
  format: "bottlerocket",
- efa: true,
  expected: bottlerocketUserData,
  expectedIsMimePart: false,
  },
  }
  for _, c := range cases {
  t.Run(c.format, func(t *testing.T) {
- actual, isMimePart, err := generateUserData(c.format, c.efa, &cluster)
+ actual, isMimePart, err := generateUserData(c.format, &cluster)
  if err != nil {
  t.Log(err)
  t.Error(err)