use cri annotations to support runtime parameters

Signed-off-by: Starnop <starnop@163.com>
AliyunContainerService · Jul 5, 2018 · 7e6a9ff · 7e6a9ff
1 parent 7caca29
commit 7e6a9ff
Show file tree

Hide file tree

Showing 14 changed files with 116 additions and 19 deletions.
diff --git a/apis/swagger.yml b/apis/swagger.yml
@@ -1964,6 +1964,9 @@ definitions:
       QuotaID:
         type: "string"
         description: "set disk quota by specified quota id, if id < 0, it means pouchd alloc a unique quota id"
+      ContainerID:
+        type: "string"
+        description: "The ID of the container"
 
   ContainerCreateResp:
     description: "response returned by daemon when container create successfully"

diff --git a/apis/types/container_config.go b/apis/types/container_config.go
diff --git a/cri/annotations/annotations.go b/cri/annotations/annotations.go
@@ -0,0 +1,19 @@
+package annotations
+
+// ContainerType values
+const (
+	// ContainerTypeSandbox represents a pod sandbox container
+	ContainerTypeSandbox = "sandbox"
+
+	// ContainerTypeContainer represents a container running within a pod
+	ContainerTypeContainer = "container"
+
+	// ContainerType is the container type (sandbox or container) annotation
+	ContainerType = "io.kubernetes.cri-o.ContainerType"
+
+	// SandboxName is the sandbox name annotation
+	SandboxName = "io.kubernetes.cri-o.SandboxName"
+
+	// KubernetesRuntime is the runtime
+	KubernetesRuntime = "io.kubernetes.runtime"
+)
diff --git a/cri/v1alpha1/cri.go b/cri/v1alpha1/cri.go
@@ -12,6 +12,7 @@ import (
 	"time"
 
 	apitypes "github.com/alibaba/pouch/apis/types"
+	anno "github.com/alibaba/pouch/cri/annotations"
 	"github.com/alibaba/pouch/daemon/config"
 	"github.com/alibaba/pouch/daemon/mgr"
 	"github.com/alibaba/pouch/pkg/errtypes"
@@ -236,6 +237,7 @@ func (c *CriManager) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		ID:        id,
 		Config:    config,
 		NetNSPath: netnsPath,
+		Runtime:   config.Annotations[anno.KubernetesRuntime],
 	}
 	c.SandboxStore.Put(sandboxMeta)
 
@@ -462,6 +464,11 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	if iSpec := config.GetImage(); iSpec != nil {
 		image = iSpec.Image
 	}
+
+	specAnnotation := make(map[string]string)
+	specAnnotation[anno.ContainerType] = anno.ContainerTypeContainer
+	specAnnotation[anno.SandboxName] = podSandboxID
+
 	createConfig := &apitypes.ContainerCreateConfig{
 		ContainerConfig: apitypes.ContainerConfig{
 			Entrypoint: config.Command,
@@ -471,9 +478,10 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
 			WorkingDir: config.WorkingDir,
 			Labels:     labels,
 			// Interactive containers:
-			OpenStdin: config.Stdin,
-			StdinOnce: config.StdinOnce,
-			Tty:       config.Tty,
+			OpenStdin:      config.Stdin,
+			StdinOnce:      config.StdinOnce,
+			Tty:            config.Tty,
+			SpecAnnotation: specAnnotation,
 		},
 		HostConfig: &apitypes.HostConfig{
 			Binds: generateMountBindings(config.GetMounts()),

diff --git a/cri/v1alpha1/cri_types.go b/cri/v1alpha1/cri_types.go
@@ -14,6 +14,9 @@ type SandboxMeta struct {
 
 	// NetNSPath is the network namespace used by the sandbox.
 	NetNSPath string
+
+	// Runtime is the runtime of sandbox
+	Runtime string
 }
 
 // Key returns sandbox's id.

diff --git a/cri/v1alpha1/cri_utils.go b/cri/v1alpha1/cri_utils.go
@@ -12,6 +12,7 @@ import (
 	"time"
 
 	apitypes "github.com/alibaba/pouch/apis/types"
+	anno "github.com/alibaba/pouch/cri/annotations"
 	"github.com/alibaba/pouch/daemon/mgr"
 	"github.com/alibaba/pouch/pkg/utils"
 	"github.com/go-openapi/strfmt"
@@ -242,6 +243,12 @@ func makeSandboxPouchConfig(config *runtime.PodSandboxConfig, image string) (*ap
 	labels[containerTypeLabelKey] = containerTypeLabelSandbox
 
 	hc := &apitypes.HostConfig{}
+
+	// Apply runtime options.
+	if annotations := config.GetAnnotations(); annotations != nil {
+		hc.Runtime = annotations[anno.KubernetesRuntime]
+	}
+
 	createConfig := &apitypes.ContainerCreateConfig{
 		ContainerConfig: apitypes.ContainerConfig{
 			Hostname: strfmt.Hostname(config.Hostname),
@@ -607,6 +614,19 @@ func applyContainerSecurityContext(lc *runtime.LinuxContainerConfig, podSandboxI
 
 // Apply Linux-specific options if applicable.
 func (c *CriManager) updateCreateConfig(createConfig *apitypes.ContainerCreateConfig, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, podSandboxID string) error {
+	// Apply runtime options.
+	if annotations := config.GetAnnotations(); annotations != nil {
+		createConfig.HostConfig.Runtime = annotations[anno.KubernetesRuntime]
+	}
+	res, err := c.SandboxStore.Get(podSandboxID)
+	if err != nil {
+		return fmt.Errorf("failed to get metadata of %q from SandboxStore: %v", podSandboxID, err)
+	}
+	sandboxMeta := res.(*SandboxMeta)
+	if sandboxMeta.Runtime != "" {
+		createConfig.HostConfig.Runtime = sandboxMeta.Runtime
+	}
+
 	if lc := config.GetLinux(); lc != nil {
 		// TODO: resource restriction.
 

diff --git a/cri/v1alpha1/cri_utils_test.go b/cri/v1alpha1/cri_utils_test.go
@@ -272,8 +272,9 @@ func Test_parseSandboxName(t *testing.T) {
 
 func Test_makeSandboxPouchConfig(t *testing.T) {
 	type args struct {
-		config *runtime.PodSandboxConfig
-		image  string
+		config    *runtime.PodSandboxConfig
+		sandboxID string
+		image     string
 	}
 	tests := []struct {
 		name    string
@@ -285,7 +286,7 @@ func Test_makeSandboxPouchConfig(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := makeSandboxPouchConfig(tt.args.config, tt.args.image)
+			got, err := makeSandboxPouchConfig(tt.args.config, tt.args.sandboxID, tt.args.image)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("makeSandboxPouchConfig() error = %v, wantErr %v", err, tt.wantErr)
 				return

diff --git a/cri/v1alpha2/cri.go b/cri/v1alpha2/cri.go
@@ -12,6 +12,7 @@ import (
 	"time"
 
 	apitypes "github.com/alibaba/pouch/apis/types"
+	anno "github.com/alibaba/pouch/cri/annotations"
 	"github.com/alibaba/pouch/daemon/config"
 	"github.com/alibaba/pouch/daemon/mgr"
 	"github.com/alibaba/pouch/pkg/errtypes"
@@ -236,6 +237,7 @@ func (c *CriManager) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		ID:        id,
 		Config:    config,
 		NetNSPath: netnsPath,
+		Runtime:   config.Annotations[anno.KubernetesRuntime],
 	}
 	c.SandboxStore.Put(sandboxMeta)
 
@@ -470,6 +472,11 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	if iSpec := config.GetImage(); iSpec != nil {
 		image = iSpec.Image
 	}
+
+	specAnnotation := make(map[string]string)
+	specAnnotation[anno.ContainerType] = anno.ContainerTypeContainer
+	specAnnotation[anno.SandboxName] = podSandboxID
+
 	createConfig := &apitypes.ContainerCreateConfig{
 		ContainerConfig: apitypes.ContainerConfig{
 			Entrypoint: config.Command,
@@ -479,9 +486,10 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
 			WorkingDir: config.WorkingDir,
 			Labels:     labels,
 			// Interactive containers:
-			OpenStdin: config.Stdin,
-			StdinOnce: config.StdinOnce,
-			Tty:       config.Tty,
+			OpenStdin:      config.Stdin,
+			StdinOnce:      config.StdinOnce,
+			Tty:            config.Tty,
+			SpecAnnotation: specAnnotation,
 		},
 		HostConfig: &apitypes.HostConfig{
 			Binds: generateMountBindings(config.GetMounts()),

diff --git a/cri/v1alpha2/cri_types.go b/cri/v1alpha2/cri_types.go
@@ -14,6 +14,9 @@ type SandboxMeta struct {
 
 	// NetNSPath is the network namespace used by the sandbox.
 	NetNSPath string
+
+	// Runtime is the runtime of sandbox
+	Runtime string
 }
 
 // Key returns sandbox's id.

diff --git a/cri/v1alpha2/cri_utils.go b/cri/v1alpha2/cri_utils.go
@@ -12,6 +12,7 @@ import (
 	"time"
 
 	apitypes "github.com/alibaba/pouch/apis/types"
+	anno "github.com/alibaba/pouch/cri/annotations"
 	"github.com/alibaba/pouch/daemon/mgr"
 	"github.com/alibaba/pouch/pkg/utils"
 
@@ -240,8 +241,13 @@ func makeSandboxPouchConfig(config *runtime.PodSandboxConfig, image string) (*ap
 	labels := makeLabels(config.GetLabels(), config.GetAnnotations())
 	// Apply a label to distinguish sandboxes from regular containers.
 	labels[containerTypeLabelKey] = containerTypeLabelSandbox
-
 	hc := &apitypes.HostConfig{}
+
+	// Apply runtime options.
+	if annotations := config.GetAnnotations(); annotations != nil {
+		hc.Runtime = annotations[anno.KubernetesRuntime]
+	}
+
 	createConfig := &apitypes.ContainerCreateConfig{
 		ContainerConfig: apitypes.ContainerConfig{
 			Hostname: strfmt.Hostname(config.Hostname),
@@ -610,6 +616,19 @@ func applyContainerSecurityContext(lc *runtime.LinuxContainerConfig, podSandboxI
 
 // Apply Linux-specific options if applicable.
 func (c *CriManager) updateCreateConfig(createConfig *apitypes.ContainerCreateConfig, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, podSandboxID string) error {
+	// Apply runtime options.
+	if annotations := config.GetAnnotations(); annotations != nil {
+		createConfig.HostConfig.Runtime = annotations[anno.KubernetesRuntime]
+	}
+	res, err := c.SandboxStore.Get(podSandboxID)
+	if err != nil {
+		return fmt.Errorf("failed to get metadata of %q from SandboxStore: %v", podSandboxID, err)
+	}
+	sandboxMeta := res.(*SandboxMeta)
+	if sandboxMeta.Runtime != "" {
+		createConfig.HostConfig.Runtime = sandboxMeta.Runtime
+	}
+
 	if lc := config.GetLinux(); lc != nil {
 		// TODO: resource restriction.
 

diff --git a/cri/v1alpha2/cri_utils_test.go b/cri/v1alpha2/cri_utils_test.go
@@ -273,8 +273,9 @@ func Test_parseSandboxName(t *testing.T) {
 
 func Test_makeSandboxPouchConfig(t *testing.T) {
 	type args struct {
-		config *runtime.PodSandboxConfig
-		image  string
+		config    *runtime.PodSandboxConfig
+		sandboxID string
+		image     string
 	}
 	tests := []struct {
 		name    string
@@ -286,7 +287,7 @@ func Test_makeSandboxPouchConfig(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := makeSandboxPouchConfig(tt.args.config, tt.args.image)
+			got, err := makeSandboxPouchConfig(tt.args.config, tt.args.sandboxID, tt.args.image)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("makeSandboxPouchConfig() error = %v, wantErr %v", err, tt.wantErr)
 				return

diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go
@@ -129,6 +129,10 @@ type ContainerMgr interface {
 
 	// Logs is used to return log created by the container.
 	Logs(ctx context.Context, name string, logsOpt *types.ContainerLogsOptions) (<-chan *logger.LogMessage, bool, error)
+
+	// GenerateID generates an ID for newly created container. We must ensure that
+	// this ID has not used yet.
+	GenerateID() (string, error)
 }
 
 // ContainerManager is the default implement of interface ContainerMgr.
@@ -248,9 +252,12 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty
 		return nil, fmt.Errorf("NetworkingConfig cannot be nil")
 	}
 
-	id, err := mgr.generateID()
-	if err != nil {
-		return nil, err
+	id := config.ContainerID
+	if id == "" {
+		id, err = mgr.GenerateID()
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if name == "" {

diff --git a/daemon/mgr/container_utils.go b/daemon/mgr/container_utils.go
@@ -69,9 +69,9 @@ func (mgr *ContainerManager) container(nameOrPrefix string) (*Container, error)
 	return nil, errors.Wrap(errtypes.ErrNotfound, "container "+nameOrPrefix)
 }
 
-// generateID generates an ID for newly created container. We must ensure that
+// GenerateID generates an ID for newly created container. We must ensure that
 // this ID has not used yet.
-func (mgr *ContainerManager) generateID() (string, error) {
+func (mgr *ContainerManager) GenerateID() (string, error) {
 	var id string
 	for {
 		id = randomid.Generate()

diff --git a/daemon/mgr/container_utils_test.go b/daemon/mgr/container_utils_test.go
@@ -30,7 +30,7 @@ func TestContainerManager_generateID(t *testing.T) {
 		Store:    store,
 	}
 
-	id, err := containerMgr.generateID()
+	id, err := containerMgr.GenerateID()
 	assert.Equal(t, len(id), 64)
 	assert.NoError(t, err)
 }