From 0f15cba87b7a346172918cb565399e8ebd05f71c Mon Sep 17 00:00:00 2001 From: yanxuean Date: Fri, 27 Jul 2018 11:44:40 +0800 Subject: [PATCH 1/4] add an example for VirtletLibvirtCPUSetting Signed-off-by: yanxuean --- .../ubuntu-vm-with-libvirt-cpusetting.yaml | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 examples/ubuntu-vm-with-libvirt-cpusetting.yaml diff --git a/examples/ubuntu-vm-with-libvirt-cpusetting.yaml b/examples/ubuntu-vm-with-libvirt-cpusetting.yaml new file mode 100644 index 000000000..11264a532 --- /dev/null +++ b/examples/ubuntu-vm-with-libvirt-cpusetting.yaml @@ -0,0 +1,43 @@ +# This example is ubuntu-vm.yaml one extended with cpu setting by libvirt domain definition +apiVersion: v1 +kind: Pod +metadata: + name: ubuntu-vm-with-nested-virtualization + annotations: + kubernetes.io/target-runtime: virtlet.cloud + VirtletSSHKeys: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost + # Set VirtletCPUModel to host-model to enable nested virtualization + VirtletCPUModel: host-model + # Set VirtletLibvirtCPUSetting to directly use cpu definition of libvirt + # for "mode": host-model, host-passthrough,custom + # for "model": Westmere, SandyBridge, etc. see file: /usr/local/share/libvirt/cpu_map.xml + # The VirtletLibvirtCPUSetting take precedence over VirtletCPUModel. + VirtletLibvirtCPUSetting: | + mode: custom + model: + value: Westmere + features: + - name: avx + policy: disable +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: extraRuntime + operator: In + values: + - virtlet + # This is the number of seconds Virtlet gives the VM to shut down cleanly. + # The default value of 30 seconds is ok for containers but probably too + # low for VM, so overriding it here is strongly advised. + terminationGracePeriodSeconds: 120 + containers: + - name: ubuntu-vm + image: virtlet.cloud/cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img + imagePullPolicy: IfNotPresent + # tty and stdin required for `kubectl attach -t` to work + tty: true + stdin: true From c236e865bea0875e5cb554f099e670f29e331791 Mon Sep 17 00:00:00 2001 From: yanxuean Date: Fri, 27 Jul 2018 11:07:15 +0800 Subject: [PATCH 2/4] support to set cpu model by libvirt xml Signed-off-by: yanxuean --- pkg/libvirttools/virtualization.go | 47 +++++++++++++++++------------- pkg/metadata/types/annotations.go | 12 ++++++++ 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/pkg/libvirttools/virtualization.go b/pkg/libvirttools/virtualization.go index 1a7707379..c42cb5626 100644 --- a/pkg/libvirttools/virtualization.go +++ b/pkg/libvirttools/virtualization.go @@ -148,28 +148,35 @@ func (ds *domainSettings) createDomain(config *types.VMConfig) *libvirtxml.Domai }, } - switch ds.cpuModel { - case types.CPUModelHostModel: - // The following enables nested virtualization. - // In case of intel processors it requires nested=1 option - // for kvm_intel module. That can be passed like this: - // modprobe kvm_intel nested=1 - domain.CPU = &libvirtxml.DomainCPU{ - Mode: types.CPUModelHostModel, - Model: &libvirtxml.DomainCPUModel{ - Fallback: "forbid", - }, - Features: []libvirtxml.DomainCPUFeature{ - { - Policy: "require", - Name: "vmx", + // Set cpu model. + // If user understand the cpu definition of libvirt, + // the user is very professional, we prior to use it. + if config.ParsedAnnotations.CPUSetting != nil { + domain.CPU = config.ParsedAnnotations.CPUSetting + } else { + switch ds.cpuModel { + case types.CPUModelHostModel: + // The following enables nested virtualization. + // In case of intel processors it requires nested=1 option + // for kvm_intel module. That can be passed like this: + // modprobe kvm_intel nested=1 + domain.CPU = &libvirtxml.DomainCPU{ + Mode: types.CPUModelHostModel, + Model: &libvirtxml.DomainCPUModel{ + Fallback: "forbid", }, - }, + Features: []libvirtxml.DomainCPUFeature{ + { + Policy: "require", + Name: "vmx", + }, + }, + } + case "": + // leave it empty + default: + glog.Warningf("Unknown value set in VIRTLET_CPU_MODEL: %q", ds.cpuModel) } - case "": - // leave it empty - default: - glog.Warningf("Unknown value set in VIRTLET_CPU_MODEL: %q", ds.cpuModel) } if ds.enableSriov { diff --git a/pkg/metadata/types/annotations.go b/pkg/metadata/types/annotations.go index d3686cedd..f19b36ad4 100644 --- a/pkg/metadata/types/annotations.go +++ b/pkg/metadata/types/annotations.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/ghodss/yaml" + libvirtxml "github.com/libvirt/libvirt-go-xml" "github.com/Mirantis/virtlet/pkg/utils" ) @@ -36,6 +37,7 @@ const ( cloudInitUserDataScriptKeyName = "VirtletCloudInitUserDataScript" cloudInitImageType = "VirtletCloudInitImageType" cpuModel = "VirtletCPUModel" + libvirtCPUSetting = "VirtletLibvirtCPUSetting" sshKeysKeyName = "VirtletSSHKeys" // CloudInitUserDataSourceKeyName is the name of user data source key in the pod annotations. CloudInitUserDataSourceKeyName = "VirtletCloudInitUserDataSource" @@ -89,6 +91,8 @@ type VirtletAnnotations struct { SSHKeys []string // DiskDriver specifies the disk driver to use. DiskDriver DiskDriverName + // CPUSetting directly specifies the cpu to use for libvirt. + CPUSetting *libvirtxml.DomainCPU } // ExternalDataLoader is a function that loads external data that's specified @@ -155,6 +159,14 @@ func loadAnnotations(ns string, podAnnotations map[string]string) (*VirtletAnnot } func (va *VirtletAnnotations) parsePodAnnotations(ns string, podAnnotations map[string]string) error { + if cpuSettingStr, found := podAnnotations[libvirtCPUSetting]; found { + var cpuSetting libvirtxml.DomainCPU + if err := yaml.Unmarshal([]byte(cpuSettingStr), &cpuSetting); err != nil { + return err + } + va.CPUSetting = &cpuSetting + } + if podAnnotations[cloudInitUserDataOverwriteKeyName] == "true" { va.UserDataOverwrite = true } From 0792c28554f094d87650d4236b1573be1a4c3863 Mon Sep 17 00:00:00 2001 From: yanxuean Date: Fri, 27 Jul 2018 12:02:27 +0800 Subject: [PATCH 3/4] need to parse VirtletCPUModel in annotation Signed-off-by: yanxuean --- pkg/metadata/types/annotations.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/metadata/types/annotations.go b/pkg/metadata/types/annotations.go index f19b36ad4..c1638c1cc 100644 --- a/pkg/metadata/types/annotations.go +++ b/pkg/metadata/types/annotations.go @@ -166,7 +166,11 @@ func (va *VirtletAnnotations) parsePodAnnotations(ns string, podAnnotations map[ } va.CPUSetting = &cpuSetting } - + + if cpuModelStr, found := podAnnotations[cpuModel]; found { + va.CPUModel = CPUModelType(cpuModelStr) + } + if podAnnotations[cloudInitUserDataOverwriteKeyName] == "true" { va.UserDataOverwrite = true } From e99612d21a024986879bcb8cc61baac3a653346a Mon Sep 17 00:00:00 2001 From: yanxuean Date: Fri, 27 Jul 2018 14:07:39 +0800 Subject: [PATCH 4/4] modify some testcase for CPUSetting Signed-off-by: yanxuean --- docs/resource_managment.md | 14 ++++++++++++++ pkg/libvirttools/TestContainerLifecycle.out.yaml | 3 +++ pkg/libvirttools/TestDomainForcedShutdown.out.yaml | 1 + 3 files changed, 18 insertions(+) diff --git a/docs/resource_managment.md b/docs/resource_managment.md index 2272852f7..b6d2cb62f 100644 --- a/docs/resource_managment.md +++ b/docs/resource_managment.md @@ -3,6 +3,20 @@ ## CPU Model By default, libvirt runs QEMU with a CPU model that doesn't support nested virtualization. It's possible to change this behavior by using `VirtletCPUModel: host-model` annotation in the pod definition. You can also use `cpuModel` value in Virtlet config to override the value globally for the cluster or for a particular subset of nodes. +If you are familiar with the cpu part in libvirt domain definition, you can use `VirtletLibvirtCPUSetting` annotation, the value is directly passed to libvirt after reading it from yaml string. It is more flexible than usage of `VirtletCPUModel` as it allows to provide more detailed configuration. +For example: +``` + annotations: + VirtletLibvirtCPUSetting: | + mode: custom + model: + value: Westmere + features: + - name: avx + policy: disable +``` +See [cpuSetting](../examples/ubuntu-vm-with-libvirt-cpusetting.yaml) for a full example. + ## Resource monitoring on the node As Kubelet uses cAdvisor to collect metrics about running containers and Virtlet doesn't create container per each VM, and instead spawns VMs inside Virtlet container. This leads to all the resource usage being lumped together and ascribed to Virtlet pod. diff --git a/pkg/libvirttools/TestContainerLifecycle.out.yaml b/pkg/libvirttools/TestContainerLifecycle.out.yaml index e7866434c..c35747d77 100755 --- a/pkg/libvirttools/TestContainerLifecycle.out.yaml +++ b/pkg/libvirttools/TestContainerLifecycle.out.yaml @@ -113,6 +113,7 @@ ParsedAnnotations: CDImageType: nocloud CPUModel: "" + CPUSetting: null DiskDriver: scsi MetaData: null SSHKeys: null @@ -165,6 +166,7 @@ ParsedAnnotations: CDImageType: nocloud CPUModel: "" + CPUSetting: null DiskDriver: scsi MetaData: null SSHKeys: null @@ -212,6 +214,7 @@ ParsedAnnotations: CDImageType: nocloud CPUModel: "" + CPUSetting: null DiskDriver: scsi MetaData: null SSHKeys: null diff --git a/pkg/libvirttools/TestDomainForcedShutdown.out.yaml b/pkg/libvirttools/TestDomainForcedShutdown.out.yaml index 9db04b38b..e537318e9 100755 --- a/pkg/libvirttools/TestDomainForcedShutdown.out.yaml +++ b/pkg/libvirttools/TestDomainForcedShutdown.out.yaml @@ -129,6 +129,7 @@ ParsedAnnotations: CDImageType: nocloud CPUModel: "" + CPUSetting: null DiskDriver: scsi MetaData: null SSHKeys: null