fix: increase shm default size and make it configurable

deploy/cloud/helm/crds/templates/nvidia.com_dynamocomponentdeployments.yaml

@@ -10241,6 +10241,18 @@ spec:
                 serviceName:
                   description: contains the name of the component
                   type: string
+                sharedMemory:
+                  description: SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size).
+                  properties:
+                    disabled:
+                      type: boolean
+                    size:
+                      anyOf:
+                        - type: integer
+                        - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                  type: object
               type: object
             status:
               description: Status reflects the current observed state of the component deployment.

deploy/cloud/helm/crds/templates/nvidia.com_dynamographdeployments.yaml

@@ -10340,6 +10340,18 @@ spec:
                       serviceName:
                         description: contains the name of the component
                         type: string
+                      sharedMemory:
+                        description: SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size).
+                        properties:
+                          disabled:
+                            type: boolean
+                          size:
+                            anyOf:
+                              - type: integer
+                              - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                        type: object
                     type: object
                   description: |-
                     Services allows per-service overrides of the component deployment settings.

deploy/cloud/operator/api/v1alpha1/common.go

@@ -44,3 +44,8 @@ type Autoscaling struct {
 	Behavior    *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"`
 	Metrics     []autoscalingv2.MetricSpec                     `json:"metrics,omitempty"`
 }
+
+type SharedMemorySpec struct {
+	Disabled bool              `json:"disabled,omitempty"`
+	Size     resource.Quantity `json:"size,omitempty"`
+}

deploy/cloud/operator/api/v1alpha1/dynamocomponentdeployment_types.go

@@ -92,6 +92,9 @@ type DynamoComponentDeploymentSharedSpec struct {
 	// Ingress config to expose the component outside the cluster (or through a service mesh).
 	Ingress *IngressSpec `json:"ingress,omitempty"`
 
+	// SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size).
+	SharedMemory *SharedMemorySpec `json:"sharedMemory,omitempty"`
+
 	// +optional
 	// ExtraPodMetadata adds labels/annotations to the created Pods.
 	ExtraPodMetadata *dynamoCommon.ExtraPodMetadata `json:"extraPodMetadata,omitempty"`

deploy/cloud/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml

@@ -10241,6 +10241,18 @@ spec:
                 serviceName:
                   description: contains the name of the component
                   type: string
+                sharedMemory:
+                  description: SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size).
+                  properties:
+                    disabled:
+                      type: boolean
+                    size:
+                      anyOf:
+                        - type: integer
+                        - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                  type: object
               type: object
             status:
               description: Status reflects the current observed state of the component deployment.

deploy/cloud/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml

@@ -10340,6 +10340,18 @@ spec:
                       serviceName:
                         description: contains the name of the component
                         type: string
+                      sharedMemory:
+                        description: SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size).
+                        properties:
+                          disabled:
+                            type: boolean
+                          size:
+                            anyOf:
+                              - type: integer
+                              - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                        type: object
                     type: object
                   description: |-
                     Services allows per-service overrides of the component deployment settings.

deploy/cloud/operator/internal/consts/consts.go

@@ -48,9 +48,11 @@ const (
 	DefaultGroveTerminationDelay = 15 * time.Minute
 
 	// Metrics related constants
-	KubeAnnotationEnableMetrics = "nvidia.com/enable-metrics"  // User-provided annotation to control metrics
-	KubeLabelMetricsEnabled     = "nvidia.com/metrics-enabled" // Controller-managed label for pod selection
-	KubeValueNameSharedMemory   = "shared-memory"
+	KubeAnnotationEnableMetrics  = "nvidia.com/enable-metrics"  // User-provided annotation to control metrics
+	KubeLabelMetricsEnabled      = "nvidia.com/metrics-enabled" // Controller-managed label for pod selection
+	KubeValueNameSharedMemory    = "shared-memory"
+	DefaultSharedMemoryMountPath = "/dev/shm"
+	DefaultSharedMemorySize      = "8Gi"
 
 	// Grove multinode role suffixes
 	GroveRoleSuffixLeader = "ldr"

deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller_test.go

@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/dynamo/common"
 	dynamoCommon "github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/dynamo/common"
 	"github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/v1alpha1"
 	commonconsts "github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/consts"
@@ -699,18 +698,18 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 								Multinode: &v1alpha1.MultinodeSpec{
 									NodeCount: 2,
 								},
-								Resources: &common.Resources{
-									Requests: &common.ResourceItem{
+								Resources: &dynamoCommon.Resources{
+									Requests: &dynamoCommon.ResourceItem{
 										CPU:    "300m",
 										Memory: "500Mi",
 									},
-									Limits: &common.ResourceItem{
+									Limits: &dynamoCommon.ResourceItem{
 										GPU:    "1",
 										Memory: "20Gi",
 										CPU:    "10",
 									},
 								},
-								ExtraPodMetadata: &common.ExtraPodMetadata{
+								ExtraPodMetadata: &dynamoCommon.ExtraPodMetadata{
 									Annotations: map[string]string{
 										"nvidia.com/annotation1": "annotation1",
 									},
@@ -793,7 +792,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 										VolumeSource: corev1.VolumeSource{
 											EmptyDir: &corev1.EmptyDirVolumeSource{
 												Medium:    corev1.StorageMediumMemory,
-												SizeLimit: resource.NewQuantity(5*1024*1024*1024, resource.BinarySI), // 5gi (calculated from memory limit / 4)
+												SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
 											},
 										},
 									},
@@ -814,7 +813,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 										VolumeMounts: []corev1.VolumeMount{
 											{
 												Name:      "shared-memory",
-												MountPath: "/dev/shm",
+												MountPath: commonconsts.DefaultSharedMemoryMountPath,
 											},
 										},
 										Resources: corev1.ResourceRequirements{
@@ -893,7 +892,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 										VolumeSource: corev1.VolumeSource{
 											EmptyDir: &corev1.EmptyDirVolumeSource{
 												Medium:    corev1.StorageMediumMemory,
-												SizeLimit: resource.NewQuantity(5*1024*1024*1024, resource.BinarySI), // 5gi (calculated from memory limit / 4)
+												SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
 											},
 										},
 									},
@@ -914,7 +913,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 										VolumeMounts: []corev1.VolumeMount{
 											{
 												Name:      "shared-memory",
-												MountPath: "/dev/shm",
+												MountPath: commonconsts.DefaultSharedMemoryMountPath,
 											},
 										},
 										Resources: corev1.ResourceRequirements{
@@ -956,8 +955,8 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 								Multinode: &v1alpha1.MultinodeSpec{
 									NodeCount: 2,
 								},
-								Resources: &common.Resources{
-									Limits: &common.ResourceItem{
+								Resources: &dynamoCommon.Resources{
+									Limits: &dynamoCommon.ResourceItem{
 										GPU: "1",
 									},
 								},
@@ -1000,8 +999,8 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 								Multinode: &v1alpha1.MultinodeSpec{
 									NodeCount: 2,
 								},
-								Resources: &common.Resources{
-									Limits: &common.ResourceItem{
+								Resources: &dynamoCommon.Resources{
+									Limits: &dynamoCommon.ResourceItem{
 										GPU: "1",
 									},
 								},

deploy/cloud/operator/internal/dynamo/graph.go

@@ -677,6 +677,8 @@ func addStandardEnvVars(container *corev1.Container, controllerConfig controller
 // GenerateBasePodSpec creates a basic PodSpec with common logic shared between controller and grove
 // Includes standard environment variables (DYNAMO_PORT, NATS_SERVER, ETCD_ENDPOINTS)
 // Deployment-specific environment merging should be handled by the caller
+//
+//nolint:gocyclo
 func GenerateBasePodSpec(
 	component *v1alpha1.DynamoComponentDeploymentOverridesSpec,
 	backendFramework BackendFramework,
@@ -778,9 +780,10 @@ func GenerateBasePodSpec(
 			MountPath: *component.PVC.MountPoint,
 		})
 	}
-	shmVolume, shmVolumeMount := generateSharedMemoryVolumeAndMount(&container.Resources)
-	volumes = append(volumes, shmVolume)
-	container.VolumeMounts = append(container.VolumeMounts, shmVolumeMount)
+	if shmVol, shmMount := generateSharedMemoryVolumeAndMount(component.SharedMemory); shmVol != nil && shmMount != nil {
+		volumes = append(volumes, *shmVol)
+		container.VolumeMounts = append(container.VolumeMounts, *shmMount)
+	}
 
 	// Apply backend-specific container modifications
 	multinodeDeployer := MultinodeDeployerFactory(multinodeDeploymentType)
@@ -1179,36 +1182,29 @@ func GenerateBasePodSpecForController(
 	return podSpec, nil
 }
 
-func generateSharedMemoryVolumeAndMount(resources *corev1.ResourceRequirements) (corev1.Volume, corev1.VolumeMount) {
-	sharedMemorySizeLimit := resource.MustParse("512Mi")
-	// Check if we have memory limits to work with
-	memoryLimit := resources.Limits[corev1.ResourceMemory]
-	if !memoryLimit.IsZero() {
-		// Use 1/4 of memory limit
-		calculatedSize := resource.NewQuantity(memoryLimit.Value()/4, resource.BinarySI)
-		// Apply bounds: minimum 512Mi, maximum 8Gi
-		minSize := resource.MustParse("512Mi")
-		maxSize := resource.MustParse("8Gi")
-
-		if calculatedSize.Cmp(minSize) > 0 && calculatedSize.Cmp(maxSize) < 0 {
-			sharedMemorySizeLimit = *calculatedSize
-		} else if calculatedSize.Cmp(maxSize) >= 0 {
-			sharedMemorySizeLimit = maxSize // Cap at maximum
+func generateSharedMemoryVolumeAndMount(spec *v1alpha1.SharedMemorySpec) (*corev1.Volume, *corev1.VolumeMount) {
+	// default: enabled=true, size=8Gi
+	size := resource.MustParse(commonconsts.DefaultSharedMemorySize)
+	if spec != nil {
+		if spec.Disabled {
+			return nil, nil
+		}
+		if !spec.Size.IsZero() {
+			size = spec.Size
 		}
-		// If calculatedSize < minSize, keep the 512Mi base
 	}
 	volume := corev1.Volume{
 		Name: commonconsts.KubeValueNameSharedMemory,
 		VolumeSource: corev1.VolumeSource{
 			EmptyDir: &corev1.EmptyDirVolumeSource{
 				Medium:    corev1.StorageMediumMemory,
-				SizeLimit: &sharedMemorySizeLimit,
+				SizeLimit: &size,
 			},
 		},
 	}
 	volumeMount := corev1.VolumeMount{
 		Name:      commonconsts.KubeValueNameSharedMemory,
-		MountPath: "/dev/shm",
+		MountPath: commonconsts.DefaultSharedMemoryMountPath,
 	}
-	return volume, volumeMount
+	return &volume, &volumeMount
 }