feat(region,host): post overlay for container volume mount

yunionio · Nov 27, 2024 · 290b8dd · 290b8dd
1 parent b94cf59
commit 290b8dd
Show file tree

Hide file tree

Showing 14 changed files with 419 additions and 110 deletions.
diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go
@@ -251,3 +251,8 @@ type ContainerResourcesSetInput struct {
 	apis.ContainerResources
 	DisableLimitCheck bool `json:"disable_limit_check"`
 }
+
+type ContainerMountVolumeMountPostOverlayInput struct {
+	Index       int                                         `json:"index"`
+	PostOverlay []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
+}
diff --git a/pkg/apis/container.go b/pkg/apis/container.go
@@ -226,14 +226,24 @@ func (o ContainerVolumeMountDiskOverlay) IsValid() error {
 	return nil
 }
 
+type ContainerVolumeMountDiskPostOverlay struct {
+	// 宿主机底层目录
+	HostLowerDir []string `json:"host_lower_dir"`
+	// 合并后要挂载到容器的目录
+	ContainerTargetDir string `json:"container_target_dir"`
+}
+
 type ContainerVolumeMountDisk struct {
-	Index           *int                             `json:"index,omitempty"`
-	Id              string                           `json:"id"`
-	SubDirectory    string                           `json:"sub_directory"`
-	StorageSizeFile string                           `json:"storage_size_file"`
-	Overlay         *ContainerVolumeMountDiskOverlay `json:"overlay"`
+	Index           *int   `json:"index,omitempty"`
+	Id              string `json:"id"`
+	SubDirectory    string `json:"sub_directory"`
+	StorageSizeFile string `json:"storage_size_file"`
+	// lower overlay 设置，disk 的 volume 会作为 upper，最终 merged 的目录会传给容器
+	Overlay *ContainerVolumeMountDiskOverlay `json:"overlay"`
 	// case insensitive feature is incompatible with overlayfs
 	CaseInsensitivePaths []string `json:"case_insensitive_paths"`
+	// 当 disk volume 挂载完后，需要 overlay 的目录设置
+	PostOverlay []*ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
 }
 
 type ContainerVolumeMountHostPathType string

diff --git a/pkg/apis/host/container.go b/pkg/apis/host/container.go
@@ -21,13 +21,14 @@ import (
 )
 
 type ContainerVolumeMountDisk struct {
-	Index                *int                                  `json:"index,omitempty"`
-	Id                   string                                `json:"id"`
-	TemplateId           string                                `json:"template_id"`
-	SubDirectory         string                                `json:"sub_directory"`
-	StorageSizeFile      string                                `json:"storage_size_file"`
-	Overlay              *apis.ContainerVolumeMountDiskOverlay `json:"overlay"`
-	CaseInsensitivePaths []string                              `json:"case_insensitive_paths"`
+	Index                *int                                        `json:"index,omitempty"`
+	Id                   string                                      `json:"id"`
+	TemplateId           string                                      `json:"template_id"`
+	SubDirectory         string                                      `json:"sub_directory"`
+	StorageSizeFile      string                                      `json:"storage_size_file"`
+	Overlay              *apis.ContainerVolumeMountDiskOverlay       `json:"overlay"`
+	CaseInsensitivePaths []string                                    `json:"case_insensitive_paths"`
+	PostOverlay          []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
 }
 
 type ContainerVolumeMountCephFS struct {

diff --git a/pkg/compute/container_drivers/volume_mount/disk.go b/pkg/compute/container_drivers/volume_mount/disk.go
@@ -135,6 +135,9 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre
 	if err := d.validateOverlay(ctx, userCred, vm, &diskObj); err != nil {
 		return nil, errors.Wrapf(err, "validate overlay")
 	}
+	if err := d.ValidatePostOverlay(vm); err != nil {
+		return nil, errors.Wrap(err, "validate post overlay")
+	}
 	return vm, nil
 }
 
@@ -184,6 +187,36 @@ func (d disk) validateOverlay(ctx context.Context, userCred mcclient.TokenCreden
 	return nil
 }
 
+func (d disk) ValidatePostOverlay(vm *apis.ContainerVolumeMount) error {
+	if len(vm.Disk.PostOverlay) == 0 {
+		return nil
+	}
+	ovs := vm.Disk.PostOverlay
+	var duplicateCtrDir string
+	for _, ov := range ovs {
+		if len(ov.HostLowerDir) == 0 {
+			return httperrors.NewNotEmptyError("host_lower_dir is required")
+		}
+		for i, hld := range ov.HostLowerDir {
+			if len(hld) == 0 {
+				return httperrors.NewNotEmptyError("host_lower_dir %d is empty", i)
+			}
+		}
+		if len(ov.ContainerTargetDir) == 0 {
+			return httperrors.NewNotEmptyError("container_target_dir is required")
+		}
+		if ov.ContainerTargetDir == duplicateCtrDir {
+			return httperrors.NewDuplicateNameError("container_target_dir", ov.ContainerTargetDir)
+		}
+		duplicateCtrDir = ov.ContainerTargetDir
+	}
+	if vm.Propagation == "" {
+		// 设置默认 propagation 为 rslave
+		vm.Propagation = apis.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER
+	}
+	return nil
+}
+
 type diskOverlayDir struct{}
 
 func newDiskOverlayDir() iDiskOverlay {

diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go
@@ -443,6 +443,7 @@ func (vm *ContainerVolumeMountRelation) toHostDiskMount(disk *apis.ContainerVolu
 		StorageSizeFile:      disk.StorageSizeFile,
 		Overlay:              disk.Overlay,
 		CaseInsensitivePaths: disk.CaseInsensitivePaths,
+		PostOverlay:          disk.PostOverlay,
 	}
 	return ret, nil
 }
@@ -987,3 +988,49 @@ func (c *SContainer) StartCommit(ctx context.Context, userCred mcclient.TokenCre
 	}
 	return task.ScheduleRun(nil)
 }
+
+func (c *SContainer) PerformAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, input *api.ContainerMountVolumeMountPostOverlayInput) (jsonutils.JSONObject, error) {
+	if !api.ContainerExitedStatus.Has(c.Status) && !api.ContainerRunningStatus.Has(c.Status) {
+		return nil, httperrors.NewInvalidStatusError("can't add post overlay on status %s", c.Status)
+	}
+	if input.Index >= len(c.Spec.VolumeMounts) {
+		return nil, httperrors.NewInputParameterError("index %d out of volume_mount size %d", input.Index, len(c.Spec.VolumeMounts))
+	}
+	vm := c.Spec.VolumeMounts[input.Index]
+	if vm.Type != apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK {
+		return nil, httperrors.NewInputParameterError("invalid volume mount type %s", vm.Type)
+	}
+
+	for _, ov := range input.PostOverlay {
+		isExist := c.isPostOverlayExist(vm, ov)
+		if isExist {
+			return nil, httperrors.NewInputParameterError("post overlay %s already exists", ov.ContainerTargetDir)
+		}
+	}
+
+	if vm.Disk.PostOverlay == nil {
+		vm.Disk.PostOverlay = []*apis.ContainerVolumeMountDiskPostOverlay{}
+	}
+
+	if api.ContainerRunningStatus.Has(c.Status) {
+		// invoke host to mount post overlay
+		// Todo
+	}
+	vm.Disk.PostOverlay = append(vm.Disk.PostOverlay, input.PostOverlay...)
+	if _, err := db.Update(c, func() error {
+		c.Spec.VolumeMounts[input.Index] = vm
+		return nil
+	}); err != nil {
+		return nil, errors.Wrapf(err, "update spec.volume_mounts[%d]", input.Index)
+	}
+	return nil, nil
+}
+
+func (c *SContainer) isPostOverlayExist(vm *apis.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) bool {
+	for _, cov := range vm.Disk.PostOverlay {
+		if ov.ContainerTargetDir == cov.ContainerTargetDir {
+			return true
+		}
+	}
+	return false
+}
diff --git a/pkg/hostman/container/volume_mount/cephfs.go b/pkg/hostman/container/volume_mount/cephfs.go
@@ -37,7 +37,9 @@ func (h cephFS) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMou
 	if err != nil {
 		return err
 	}
-	procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output()
+	if err := EnsureDir(dir); err != nil {
+		return errors.Wrap(err, "EnsureDir")
+	}
 	options := fmt.Sprintf("name=%s,secret=%s", vm.CephFS.Name, vm.CephFS.Secret)
 	if vm.ReadOnly {
 		options += ",ro"