support to set cpu model with more detail

docs/resource_managment.md

@@ -3,6 +3,20 @@
 ## CPU Model
 By default, libvirt runs QEMU with a CPU model that doesn't support nested virtualization. It's possible to change this behavior by using `VirtletCPUModel: host-model` annotation in the pod definition. You can also use `cpuModel` value in Virtlet config to override the value globally for the cluster or for a particular subset of nodes.
 
+If you are familiar with the cpu part in libvirt domain definition, you can use `VirtletLibvirtCPUSetting` annotation, the value is directly passed to libvirt after reading it from yaml string. It is more flexible than usage of `VirtletCPUModel` as it allows to provide more detailed configuration.
+For example:
+```
+ annotations:
+ VirtletLibvirtCPUSetting: |
+ mode: custom
+ model:
+ value: Westmere
+ features:
+ - name: avx
+ policy: disable
+```
+See [cpuSetting](../examples/ubuntu-vm-with-libvirt-cpusetting.yaml) for a full example.
+
 ## Resource monitoring on the node
 As Kubelet uses cAdvisor to collect metrics about running containers and Virtlet doesn't create container per each VM, and instead spawns VMs inside Virtlet container. This leads to all the resource usage being lumped together and ascribed to Virtlet pod.
 

examples/ubuntu-vm-with-libvirt-cpusetting.yaml

@@ -0,0 +1,43 @@
+# This example is ubuntu-vm.yaml one extended with cpu setting by libvirt domain definition
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ubuntu-vm-with-nested-virtualization
+ annotations:
+ kubernetes.io/target-runtime: virtlet.cloud
+ VirtletSSHKeys: |
+ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
+ # Set VirtletCPUModel to host-model to enable nested virtualization
+ VirtletCPUModel: host-model
+ # Set VirtletLibvirtCPUSetting to directly use cpu definition of libvirt
+ # for "mode": host-model, host-passthrough,custom
+ # for "model": Westmere, SandyBridge, etc. see file: /usr/local/share/libvirt/cpu_map.xml
+ # The VirtletLibvirtCPUSetting take precedence over VirtletCPUModel.
+ VirtletLibvirtCPUSetting: |
+ mode: custom
+ model:
+ value: Westmere
+ features:
+ - name: avx
+ policy: disable
+spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: extraRuntime
+ operator: In
+ values:
+ - virtlet
+ # This is the number of seconds Virtlet gives the VM to shut down cleanly.
+ # The default value of 30 seconds is ok for containers but probably too
+ # low for VM, so overriding it here is strongly advised.
+ terminationGracePeriodSeconds: 120
+ containers:
+ - name: ubuntu-vm
+ image: virtlet.cloud/cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
+ imagePullPolicy: IfNotPresent
+ # tty and stdin required for `kubectl attach -t` to work
+ tty: true
+ stdin: true

pkg/libvirttools/TestContainerLifecycle.out.yaml

@@ -113,6 +113,7 @@
  ParsedAnnotations:
  CDImageType: nocloud
  CPUModel: ""
+ CPUSetting: null
  DiskDriver: scsi
  MetaData: null
  SSHKeys: null
@@ -165,6 +166,7 @@
  ParsedAnnotations:
  CDImageType: nocloud
  CPUModel: ""
+ CPUSetting: null
  DiskDriver: scsi
  MetaData: null
  SSHKeys: null
@@ -212,6 +214,7 @@
  ParsedAnnotations:
  CDImageType: nocloud
  CPUModel: ""
+ CPUSetting: null
  DiskDriver: scsi
  MetaData: null
  SSHKeys: null

pkg/libvirttools/TestDomainForcedShutdown.out.yaml

@@ -129,6 +129,7 @@
  ParsedAnnotations:
  CDImageType: nocloud
  CPUModel: ""
+ CPUSetting: null
  DiskDriver: scsi
  MetaData: null
  SSHKeys: null

pkg/libvirttools/virtualization.go

@@ -148,28 +148,35 @@ func (ds *domainSettings) createDomain(config *types.VMConfig) *libvirtxml.Domai
  },
  }
 
- switch ds.cpuModel {
- case types.CPUModelHostModel:
- // The following enables nested virtualization.
- // In case of intel processors it requires nested=1 option
- // for kvm_intel module. That can be passed like this:
- // modprobe kvm_intel nested=1
- domain.CPU = &libvirtxml.DomainCPU{
- Mode: types.CPUModelHostModel,
- Model: &libvirtxml.DomainCPUModel{
- Fallback: "forbid",
- },
- Features: []libvirtxml.DomainCPUFeature{
- {
- Policy: "require",
- Name: "vmx",
+ // Set cpu model.
+ // If user understand the cpu definition of libvirt,
+ // the user is very professional, we prior to use it.
+ if config.ParsedAnnotations.CPUSetting != nil {
+ domain.CPU = config.ParsedAnnotations.CPUSetting
+ } else {
+ switch ds.cpuModel {
+ case types.CPUModelHostModel:
+ // The following enables nested virtualization.
+ // In case of intel processors it requires nested=1 option
+ // for kvm_intel module. That can be passed like this:
+ // modprobe kvm_intel nested=1
+ domain.CPU = &libvirtxml.DomainCPU{
+ Mode: types.CPUModelHostModel,
+ Model: &libvirtxml.DomainCPUModel{
+ Fallback: "forbid",
  },
- },
+ Features: []libvirtxml.DomainCPUFeature{
+ {
+ Policy: "require",
+ Name: "vmx",
+ },
+ },
+ }
+ case "":
+ // leave it empty
+ default:
+ glog.Warningf("Unknown value set in VIRTLET_CPU_MODEL: %q", ds.cpuModel)
  }
- case "":
- // leave it empty
- default:
- glog.Warningf("Unknown value set in VIRTLET_CPU_MODEL: %q", ds.cpuModel)
  }
 
  if ds.enableSriov {

pkg/metadata/types/annotations.go

@@ -22,6 +22,7 @@ import (
  "strings"
 
  "github.com/ghodss/yaml"
+ libvirtxml "github.com/libvirt/libvirt-go-xml"
 
  "github.com/Mirantis/virtlet/pkg/utils"
 )
@@ -36,6 +37,7 @@ const (
  cloudInitUserDataScriptKeyName = "VirtletCloudInitUserDataScript"
  cloudInitImageType = "VirtletCloudInitImageType"
  cpuModel = "VirtletCPUModel"
+ libvirtCPUSetting = "VirtletLibvirtCPUSetting"
  sshKeysKeyName = "VirtletSSHKeys"
  // CloudInitUserDataSourceKeyName is the name of user data source key in the pod annotations.
  CloudInitUserDataSourceKeyName = "VirtletCloudInitUserDataSource"
@@ -89,6 +91,8 @@ type VirtletAnnotations struct {
  SSHKeys []string
  // DiskDriver specifies the disk driver to use.
  DiskDriver DiskDriverName
+ // CPUSetting directly specifies the cpu to use for libvirt.
+ CPUSetting *libvirtxml.DomainCPU
 }
 
 // ExternalDataLoader is a function that loads external data that's specified
@@ -155,6 +159,18 @@ func loadAnnotations(ns string, podAnnotations map[string]string) (*VirtletAnnot
 }
 
 func (va *VirtletAnnotations) parsePodAnnotations(ns string, podAnnotations map[string]string) error {
+ if cpuSettingStr, found := podAnnotations[libvirtCPUSetting]; found {
+ var cpuSetting libvirtxml.DomainCPU
+ if err := yaml.Unmarshal([]byte(cpuSettingStr), &cpuSetting); err != nil {
+ return err
+ }
+ va.CPUSetting = &cpuSetting
+ }
+
+ if cpuModelStr, found := podAnnotations[cpuModel]; found {
+ va.CPUModel = CPUModelType(cpuModelStr)
+ }
+
  if podAnnotations[cloudInitUserDataOverwriteKeyName] == "true" {
  va.UserDataOverwrite = true
  }