use cri annotations to support runtime parameters

Signed-off-by: Starnop <starnop@163.com>
AliyunContainerService · Jun 25, 2018 · 9fa1ee5 · 9fa1ee5
1 parent a8b169c
commit 9fa1ee5
Show file tree

Hide file tree

Showing 12 changed files with 126 additions and 33 deletions.
diff --git a/apis/swagger.yml b/apis/swagger.yml
@@ -1964,6 +1964,9 @@ definitions:
  QuotaID:
  type: "string"
  description: "set disk quota by specified quota id, if id < 0, it means pouchd alloc a unique quota id"
+ ContainerID:
+ type: "string"
+ description: "The ID of the container"
 
  ContainerCreateResp:
  description: "response returned by daemon when container create successfully"

diff --git a/apis/types/container_config.go b/apis/types/container_config.go
diff --git a/cri/v1alpha1/cri.go b/cri/v1alpha1/cri.go
@@ -12,6 +12,7 @@ import (
  "time"
 
  apitypes "github.com/alibaba/pouch/apis/types"
+ anno "github.com/alibaba/pouch/daemon/annotations"
  "github.com/alibaba/pouch/daemon/config"
  "github.com/alibaba/pouch/daemon/mgr"
  "github.com/alibaba/pouch/pkg/errtypes"
@@ -161,19 +162,23 @@ func (c *CriManager) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
  return nil, err
  }
 
+ id, err := c.ContainerMgr.GenerateID()
+ if err != nil {
+ return nil, err
+ }
+
  // Step 2: Create the sandbox container.
- createConfig, err := makeSandboxPouchConfig(config, image)
+ createConfig, err := makeSandboxPouchConfig(config, id, image)
  if err != nil {
  return nil, fmt.Errorf("failed to make sandbox pouch config for pod %q: %v", config.Metadata.Name, err)
  }
 
  sandboxName := makeSandboxName(config)
 
- createResp, err := c.ContainerMgr.Create(ctx, sandboxName, createConfig)
+ _, err = c.ContainerMgr.Create(ctx, sandboxName, createConfig)
  if err != nil {
  return nil, fmt.Errorf("failed to create a sandbox for pod %q: %v", config.Metadata.Name, err)
  }
- id := createResp.ID
  defer func() {
  // If running sandbox failed, clean up the container.
  if retErr != nil {
@@ -459,6 +464,11 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
  if iSpec := config.GetImage(); iSpec != nil {
  image = iSpec.Image
  }
+
+ specAnnotation := make(map[string]string)
+ specAnnotation[anno.ContainerType] = anno.ContainerTypeSandbox
+ specAnnotation[anno.SandboxID] = podSandboxID
+
  createConfig := &apitypes.ContainerCreateConfig{
  ContainerConfig: apitypes.ContainerConfig{
  Entrypoint: config.Command,
@@ -468,9 +478,10 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
  WorkingDir: config.WorkingDir,
  Labels: labels,
  // Interactive containers:
- OpenStdin: config.Stdin,
- StdinOnce: config.StdinOnce,
- Tty: config.Tty,
+ OpenStdin: config.Stdin,
+ StdinOnce: config.StdinOnce,
+ Tty: config.Tty,
+ SpecAnnotation: specAnnotation,
  },
  HostConfig: &apitypes.HostConfig{
  Binds: generateMountBindings(config.GetMounts()),

diff --git a/cri/v1alpha1/cri_utils.go b/cri/v1alpha1/cri_utils.go
@@ -12,6 +12,7 @@ import (
  "time"
 
  apitypes "github.com/alibaba/pouch/apis/types"
+ anno "github.com/alibaba/pouch/daemon/annotations"
  "github.com/alibaba/pouch/daemon/mgr"
  "github.com/alibaba/pouch/pkg/utils"
  "github.com/go-openapi/strfmt"
@@ -235,18 +236,30 @@ func applySandboxLinuxOptions(hc *apitypes.HostConfig, lc *runtime.LinuxPodSandb
 }
 
 // makeSandboxPouchConfig returns apitypes.ContainerCreateConfig based on runtimeapi.PodSandboxConfig.
-func makeSandboxPouchConfig(config *runtime.PodSandboxConfig, image string) (*apitypes.ContainerCreateConfig, error) {
+func makeSandboxPouchConfig(config *runtime.PodSandboxConfig, sandboxID string, image string) (*apitypes.ContainerCreateConfig, error) {
  // Merge annotations and labels because pouch supports only labels.
  labels := makeLabels(config.GetLabels(), config.GetAnnotations())
  // Apply a label to distinguish sandboxes from regular containers.
  labels[containerTypeLabelKey] = containerTypeLabelSandbox
 
  hc := &apitypes.HostConfig{}
+
+ // Apply runtime options.
+ if annotations := config.GetAnnotations(); annotations != nil {
+ hc.Runtime = annotations[anno.KubernetesRuntime]
+ }
+
+ specAnnotation := make(map[string]string)
+ specAnnotation[anno.ContainerType] = anno.ContainerTypeSandbox
+ specAnnotation[anno.SandboxID] = sandboxID
+
  createConfig := &apitypes.ContainerCreateConfig{
  ContainerConfig: apitypes.ContainerConfig{
- Hostname: strfmt.Hostname(config.Hostname),
- Image: image,
- Labels: labels,
+ Hostname: strfmt.Hostname(config.Hostname),
+ Image: image,
+ Labels: labels,
+ ContainerID: sandboxID,
+ SpecAnnotation: specAnnotation,
  },
  HostConfig: hc,
  NetworkingConfig: &apitypes.NetworkingConfig{},
@@ -607,6 +620,11 @@ func applyContainerSecurityContext(lc *runtime.LinuxContainerConfig, podSandboxI
 
 // Apply Linux-specific options if applicable.
 func (c *CriManager) updateCreateConfig(createConfig *apitypes.ContainerCreateConfig, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, podSandboxID string) error {
+ // Apply runtime options.
+ if annotations := config.GetAnnotations(); annotations != nil {
+ createConfig.HostConfig.Runtime = annotations[anno.KubernetesRuntime]
+ }
+
  if lc := config.GetLinux(); lc != nil {
  // TODO: resource restriction.
 

diff --git a/cri/v1alpha1/cri_utils_test.go b/cri/v1alpha1/cri_utils_test.go
@@ -272,8 +272,9 @@ func Test_parseSandboxName(t *testing.T) {
 
 func Test_makeSandboxPouchConfig(t *testing.T) {
  type args struct {
- config *runtime.PodSandboxConfig
- image string
+ config *runtime.PodSandboxConfig
+ sandboxID string
+ image string
  }
  tests := []struct {
  name string
@@ -285,7 +286,7 @@ func Test_makeSandboxPouchConfig(t *testing.T) {
  }
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- got, err := makeSandboxPouchConfig(tt.args.config, tt.args.image)
+ got, err := makeSandboxPouchConfig(tt.args.config, tt.args.sandboxID, tt.args.image)
  if (err != nil) != tt.wantErr {
  t.Errorf("makeSandboxPouchConfig() error = %v, wantErr %v", err, tt.wantErr)
  return

diff --git a/cri/v1alpha2/cri.go b/cri/v1alpha2/cri.go
@@ -12,6 +12,7 @@ import (
  "time"
 
  apitypes "github.com/alibaba/pouch/apis/types"
+ anno "github.com/alibaba/pouch/daemon/annotations"
  "github.com/alibaba/pouch/daemon/config"
  "github.com/alibaba/pouch/daemon/mgr"
  "github.com/alibaba/pouch/pkg/errtypes"
@@ -161,19 +162,23 @@ func (c *CriManager) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
  return nil, err
  }
 
+ id, err := c.ContainerMgr.GenerateID()
+ if err != nil {
+ return nil, err
+ }
+
  // Step 2: Create the sandbox container.
- createConfig, err := makeSandboxPouchConfig(config, image)
+ createConfig, err := makeSandboxPouchConfig(config, id, image)
  if err != nil {
  return nil, fmt.Errorf("failed to make sandbox pouch config for pod %q: %v", config.Metadata.Name, err)
  }
 
  sandboxName := makeSandboxName(config)
 
- createResp, err := c.ContainerMgr.Create(ctx, sandboxName, createConfig)
+ _, err = c.ContainerMgr.Create(ctx, sandboxName, createConfig)
  if err != nil {
  return nil, fmt.Errorf("failed to create a sandbox for pod %q: %v", config.Metadata.Name, err)
  }
- id := createResp.ID
  defer func() {
  // If running sandbox failed, clean up the container.
  if retErr != nil {
@@ -467,6 +472,11 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
  if iSpec := config.GetImage(); iSpec != nil {
  image = iSpec.Image
  }
+
+ specAnnotation := make(map[string]string)
+ specAnnotation[anno.ContainerType] = anno.ContainerTypeSandbox
+ specAnnotation[anno.SandboxID] = podSandboxID
+
  createConfig := &apitypes.ContainerCreateConfig{
  ContainerConfig: apitypes.ContainerConfig{
  Entrypoint: config.Command,
@@ -476,9 +486,10 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
  WorkingDir: config.WorkingDir,
  Labels: labels,
  // Interactive containers:
- OpenStdin: config.Stdin,
- StdinOnce: config.StdinOnce,
- Tty: config.Tty,
+ OpenStdin: config.Stdin,
+ StdinOnce: config.StdinOnce,
+ Tty: config.Tty,
+ SpecAnnotation: specAnnotation,
  },
  HostConfig: &apitypes.HostConfig{
  Binds: generateMountBindings(config.GetMounts()),

diff --git a/cri/v1alpha2/cri_utils.go b/cri/v1alpha2/cri_utils.go
@@ -12,6 +12,7 @@ import (
  "time"
 
  apitypes "github.com/alibaba/pouch/apis/types"
+ anno "github.com/alibaba/pouch/daemon/annotations"
  "github.com/alibaba/pouch/daemon/mgr"
  "github.com/alibaba/pouch/pkg/utils"
 
@@ -235,18 +236,29 @@ func applySandboxLinuxOptions(hc *apitypes.HostConfig, lc *runtime.LinuxPodSandb
 }
 
 // makeSandboxPouchConfig returns apitypes.ContainerCreateConfig based on runtime.PodSandboxConfig.
-func makeSandboxPouchConfig(config *runtime.PodSandboxConfig, image string) (*apitypes.ContainerCreateConfig, error) {
+func makeSandboxPouchConfig(config *runtime.PodSandboxConfig, sandboxID string, image string) (*apitypes.ContainerCreateConfig, error) {
  // Merge annotations and labels because pouch supports only labels.
  labels := makeLabels(config.GetLabels(), config.GetAnnotations())
  // Apply a label to distinguish sandboxes from regular containers.
  labels[containerTypeLabelKey] = containerTypeLabelSandbox
-
  hc := &apitypes.HostConfig{}
+
+ // Apply runtime options.
+ if annotations := config.GetAnnotations(); annotations != nil {
+ hc.Runtime = annotations[anno.KubernetesRuntime]
+ }
+
+ specAnnotation := make(map[string]string)
+ specAnnotation[anno.ContainerType] = anno.ContainerTypeSandbox
+ specAnnotation[anno.SandboxID] = sandboxID
+
  createConfig := &apitypes.ContainerCreateConfig{
  ContainerConfig: apitypes.ContainerConfig{
- Hostname: strfmt.Hostname(config.Hostname),
- Image: image,
- Labels: labels,
+ Hostname: strfmt.Hostname(config.Hostname),
+ Image: image,
+ Labels: labels,
+ ContainerID: sandboxID,
+ SpecAnnotation: specAnnotation,
  },
  HostConfig: hc,
  NetworkingConfig: &apitypes.NetworkingConfig{},
@@ -610,6 +622,11 @@ func applyContainerSecurityContext(lc *runtime.LinuxContainerConfig, podSandboxI
 
 // Apply Linux-specific options if applicable.
 func (c *CriManager) updateCreateConfig(createConfig *apitypes.ContainerCreateConfig, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, podSandboxID string) error {
+ // Apply runtime options.
+ if annotations := config.GetAnnotations(); annotations != nil {
+ createConfig.HostConfig.Runtime = annotations[anno.KubernetesRuntime]
+ }
+
  if lc := config.GetLinux(); lc != nil {
  // TODO: resource restriction.
 

diff --git a/cri/v1alpha2/cri_utils_test.go b/cri/v1alpha2/cri_utils_test.go
@@ -273,8 +273,9 @@ func Test_parseSandboxName(t *testing.T) {
 
 func Test_makeSandboxPouchConfig(t *testing.T) {
  type args struct {
- config *runtime.PodSandboxConfig
- image string
+ config *runtime.PodSandboxConfig
+ sandboxID string
+ image string
  }
  tests := []struct {
  name string
@@ -286,7 +287,7 @@ func Test_makeSandboxPouchConfig(t *testing.T) {
  }
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- got, err := makeSandboxPouchConfig(tt.args.config, tt.args.image)
+ got, err := makeSandboxPouchConfig(tt.args.config, tt.args.sandboxID, tt.args.image)
  if (err != nil) != tt.wantErr {
  t.Errorf("makeSandboxPouchConfig() error = %v, wantErr %v", err, tt.wantErr)
  return

diff --git a/daemon/annotations/annotations.go b/daemon/annotations/annotations.go
@@ -0,0 +1,19 @@
+package annotations
+
+// ContainerType values
+const (
+ // ContainerTypeSandbox represents a pod sandbox container
+ ContainerTypeSandbox = "sandbox"
+
+ // ContainerTypeContainer represents a container running within a pod
+ ContainerTypeContainer = "container"
+
+ // ContainerType is the container type (sandbox or container) annotation
+ ContainerType = "io.kubernetes.cri.container-type"
+
+ // SandboxID is the sandbox ID annotation
+ SandboxID = "io.kubernetes.cri.sandbox-id"
+
+ // KubernetesRuntime is the runtime
+ KubernetesRuntime = "io.kubernetes.runtime"
+)
diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go
@@ -129,6 +129,10 @@ type ContainerMgr interface {
 
  // Logs is used to return log created by the container.
  Logs(ctx context.Context, name string, logsOpt *types.ContainerLogsOptions) (<-chan *logger.LogMessage, bool, error)
+
+ // GenerateID generates an ID for newly created container. We must ensure that
+ // this ID has not used yet.
+ GenerateID() (string, error)
 }
 
 // ContainerManager is the default implement of interface ContainerMgr.
@@ -248,9 +252,12 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty
  return nil, fmt.Errorf("NetworkingConfig cannot be nil")
  }
 
- id, err := mgr.generateID()
- if err != nil {
- return nil, err
+ id := config.ContainerID
+ if id == "" {
+ id, err = mgr.GenerateID()
+ if err != nil {
+ return nil, err
+ }
  }
 
  if name == "" {

diff --git a/daemon/mgr/container_utils.go b/daemon/mgr/container_utils.go
@@ -69,9 +69,9 @@ func (mgr *ContainerManager) container(nameOrPrefix string) (*Container, error)
  return nil, errors.Wrap(errtypes.ErrNotfound, "container "+nameOrPrefix)
 }
 
-// generateID generates an ID for newly created container. We must ensure that
+// GenerateID generates an ID for newly created container. We must ensure that
 // this ID has not used yet.
-func (mgr *ContainerManager) generateID() (string, error) {
+func (mgr *ContainerManager) GenerateID() (string, error) {
  var id string
  for {
  id = randomid.Generate()

diff --git a/daemon/mgr/container_utils_test.go b/daemon/mgr/container_utils_test.go
@@ -30,7 +30,7 @@ func TestContainerManager_generateID(t *testing.T) {
  Store: store,
  }
 
- id, err := containerMgr.generateID()
+ id, err := containerMgr.GenerateID()
  assert.Equal(t, len(id), 64)
  assert.NoError(t, err)
 }