Add persistent root filesystem

If volumeDevices has an entry with 'devicePath: /', it's used as a
persistent root filesystem for the VM. It gets overwritten
again if the image digest changes.
See examples/cirros-vm-persistent-rootfs.yaml

examples/cirros-vm-persistent-rootfs.yaml

@@ -0,0 +1,80 @@
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+ name: local-block-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ volumeMode: Block
+ storageClassName: local-storage
+ resources:
+ requests:
+ storage: 100Mi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: local-block-pv
+spec:
+ capacity:
+ storage: 100Mi
+ accessModes:
+ - ReadWriteOnce
+ persistentVolumeReclaimPolicy: Retain
+ storageClassName: local-storage
+ volumeMode: Block
+ local:
+ # set up with:
+ # docker exec kube-node-1 /bin/bash -c 'dd if=/dev/zero of=/rawtest bs=1M count=1000 && losetup -f /rawtest --show'
+ path: /dev/loop0
+ claimRef:
+ name: local-block-pvc
+ namespace: default
+ nodeAffinity:
+ required:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - kube-node-1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: cirros-vm-p
+ annotations:
+ kubernetes.io/target-runtime: virtlet.cloud
+ # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason
+ VirtletDiskDriver: virtio
+ VirtletSSHKeys: |
+ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
+spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: extraRuntime
+ operator: In
+ values:
+ - virtlet
+ # This is the number of seconds Virtlet gives the VM to shut down cleanly.
+ # The default value of 30 seconds is ok for containers but probably too
+ # low for VM, so overriding it here is strongly advised.
+ terminationGracePeriodSeconds: 120
+ containers:
+ - name: cirros-vm
+ image: virtlet.cloud/cirros
+ imagePullPolicy: IfNotPresent
+ # tty and stdin required for `kubectl attach -t` to work
+ tty: true
+ stdin: true
+ volumeDevices:
+ - devicePath: /
+ name: testpvc
+ volumes:
+ - name: testpvc
+ persistentVolumeClaim:
+ claimName: local-block-pvc

pkg/image/fake/store.go

@@ -119,13 +119,13 @@ func (s *FakeStore) GC() error {
  return nil
 }
 
-// GetImagePathAndVirtualSize implements GC method of Store interface.
-func (s *FakeStore) GetImagePathAndVirtualSize(imageName string) (string, uint64, error) {
+// GetImagePathDigestAndVirtualSize implements GetImagePathDigestAndVirtualSize method of Store interface.
+func (s *FakeStore) GetImagePathDigestAndVirtualSize(imageName string) (string, digest.Digest, uint64, error) {
  img, found := s.images[imageName]
  if !found {
- return "", 0, fmt.Errorf("image not found: %q", imageName)
+ return "", "", 0, fmt.Errorf("image not found: %q", imageName)
  }
- return img.Path, img.Size, nil
+ return img.Path, digest.Digest(img.Digest), img.Size, nil
 }
 
 // SetRefGetter implements SetRefGetter method of Store interface.

pkg/image/image.go

@@ -81,10 +81,10 @@ type Store interface {
  // GC removes all unused or partially downloaded images.
  GC() error
 
- // GetImagePathAndVirtualSize returns the path to image data
- // and virtual size for the specified image. It accepts
- // an image reference or a digest.
- GetImagePathAndVirtualSize(ref string) (string, uint64, error)
+ // GetImagePathDigestAndVirtualSize returns the path to image
+ // data, the digest and the virtual size for the specified
+ // image. It accepts an image reference or a digest.
+ GetImagePathDigestAndVirtualSize(ref string) (string, digest.Digest, uint64, error)
 
  // SetRefGetter sets a function that will be used to determine
  // the set of images that are currently in use.
@@ -472,30 +472,33 @@ func (s *FileStore) GC() error {
  return nil
 }
 
-// GetImagePathAndVirtualSize implements GC method of Store interface.
-func (s *FileStore) GetImagePathAndVirtualSize(ref string) (string, uint64, error) {
+// GetImagePathDigestAndVirtualSize implements GetImagePathDigestAndVirtualSize method of Store interface.
+func (s *FileStore) GetImagePathDigestAndVirtualSize(ref string) (string, digest.Digest, uint64, error) {
  s.Lock()
  defer s.Unlock()
- glog.V(3).Infof("GetImagePathAndVirtualSize(): %q", ref)
+ glog.V(3).Infof("GetImagePathDigestAndVirtualSize(): %q", ref)
 
  var pathViaDigest, pathViaName string
  // parsing digest as ref gives bad results
- if d, err := digest.Parse(ref); err == nil {
+ d, err := digest.Parse(ref)
+ if err == nil {
  if d.Algorithm() != digest.SHA256 {
- return "", 0, fmt.Errorf("bad image digest (need sha256): %q", d)
+ return "", "", 0, fmt.Errorf("bad image digest (need sha256): %q", d)
  }
  pathViaDigest = s.dataFileName(d.Hex())
  } else {
  parsed, err := reference.Parse(ref)
  if err != nil {
- return "", 0, fmt.Errorf("bad image reference %q: %v", ref, err)
+ return "", "", 0, fmt.Errorf("bad image reference %q: %v", ref, err)
  }
 
+ d = ""
  if digested, ok := parsed.(reference.Digested); ok {
  if digested.Digest().Algorithm() != digest.SHA256 {
- return "", 0, fmt.Errorf("bad image digest (need sha256): %q", digested.Digest())
+ return "", "", 0, fmt.Errorf("bad image digest (need sha256): %q", digested.Digest())
  }
- pathViaDigest = s.dataFileName(digested.Digest().Hex())
+ d = digested.Digest()
+ pathViaDigest = s.dataFileName(d.Hex())
  }
 
  if named, ok := parsed.(reference.Named); ok && named.Name() != "" {
@@ -504,35 +507,36 @@ func (s *FileStore) GetImagePathAndVirtualSize(ref string) (string, uint64, erro
  glog.Warningf("error reading link %q: %v", pathViaName, err)
  } else {
  pathViaName = filepath.Join(s.linkDir(), pathViaName)
+ d = digest.NewDigestFromHex(string(digest.SHA256), filepath.Base(pathViaName))
  }
  }
  }
 
  path := pathViaDigest
  switch {
  case pathViaDigest == "" && pathViaName == "":
- return "", 0, fmt.Errorf("bad image reference %q", ref)
+ return "", "", 0, fmt.Errorf("bad image reference %q", ref)
  case pathViaDigest == "":
  path = pathViaName
  case pathViaName != "":
  fi1, err := os.Stat(pathViaName)
  if err != nil {
- return "", 0, err
+ return "", "", 0, err
  }
  fi2, err := os.Stat(pathViaDigest)
  if err != nil {
- return "", 0, err
+ return "", "", 0, err
  }
  if !os.SameFile(fi1, fi2) {
- return "", 0, fmt.Errorf("digest / name path mismatch: %q vs %q", pathViaDigest, pathViaName)
+ return "", "", 0, fmt.Errorf("digest / name path mismatch: %q vs %q", pathViaDigest, pathViaName)
  }
  }
 
  vsize, err := s.vsizeFunc(path)
  if err != nil {
- return "", 0, fmt.Errorf("error getting image size for %q: %v", path, err)
+ return "", "", 0, fmt.Errorf("error getting image size for %q: %v", path, err)
  }
- return path, vsize, nil
+ return path, d, vsize, nil
 }
 
 // SetRefGetter implements SetRefGetter method of Store interface.

pkg/image/image_test.go

@@ -187,9 +187,13 @@ func (tst *ifsTester) verifyListImages(filter string, expectedImages ...*Image)
 }
 
 func (tst *ifsTester) verifyImage(ref string, expectedContents string) {
- if path, vsize, err := tst.store.GetImagePathAndVirtualSize(ref); err != nil {
+ if path, digest, vsize, err := tst.store.GetImagePathDigestAndVirtualSize(ref); err != nil {
  tst.t.Errorf("GetImagePathAndVirtualSize(): %v", err)
  } else {
+ expectedDigest := "sha256:" + sha256str(expectedContents)
+ if string(digest) != expectedDigest {
+ tst.t.Errorf("bad digest: %s instead of %s", digest, expectedDigest)
+ }
  tst.verifyFileContents(path, expectedContents)
  expectedVirtualSize := uint64(len(expectedContents)) + 1000
  if vsize != expectedVirtualSize {

pkg/libvirttools/TestContainerLifecycle.out.yaml

@@ -1,6 +1,6 @@
 - name: 'domain conn: ListDomains'
  value: []
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainDefinitions__ceph_flexvolume.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainDefinitions__cloud-init.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainDefinitions__cloud-init_with_user_data.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainDefinitions__persistent_rootfs.out.yaml

@@ -0,0 +1,87 @@
+- name: GetImagePathDigestAndVirtualSize
+ value: fake/image1
+- name: CMD
+ value:
+ cmd: blockdev --getsz /fakedev/69eec606-0493-5825-73a4-c5e0c0236155/volumeDevices/kubernetes.io~local-volume/root
+ stdout: "1000"
+- name: CMD
+ value:
+ cmd: dmsetup create virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155
+ stdin: |
+ 0 999 linear /fakedev/69eec606-0493-5825-73a4-c5e0c0236155/volumeDevices/kubernetes.io~local-volume/root 1
+- name: CMD
+ value:
+ cmd: qemu-img convert -O raw /fake/volume/path /dev/mapper/virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155
+- name: 'domain conn: DefineDomain'
+ value: |-
+ <domain type="kvm">
+ <name>virtlet-231700d5-c9a6-container1</name>
+ <uuid>231700d5-c9a6-5a49-738d-99a954c51550</uuid>
+ <memory unit="MiB">1024</memory>
+ <vcpu>1</vcpu>
+ <cputune>
+ <shares>0</shares>
+ <period>0</period>
+ <quota>0</quota>
+ </cputune>
+ <os>
+ <type>hvm</type>
+ <boot dev="hd"></boot>
+ </os>
+ <features>
+ <acpi></acpi>
+ </features>
+ <on_poweroff>destroy</on_poweroff>
+ <on_reboot>restart</on_reboot>
+ <on_crash>restart</on_crash>
+ <devices>
+ <emulator>/vmwrapper</emulator>
+ <disk type="block" device="disk">
+ <driver name="qemu" type="raw"></driver>
+ <source dev="/dev/mapper/virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155"></source>
+ <target dev="sda" bus="scsi"></target>
+ <address type="drive" controller="0" bus="0" target="0" unit="0"></address>
+ </disk>
+ <disk type="file" device="cdrom">
+ <driver name="qemu" type="raw"></driver>
+ <source file="/var/lib/virtlet/config/config-231700d5-c9a6-5a49-738d-99a954c51550.iso"></source>
+ <target dev="sdb" bus="scsi"></target>
+ <readonly></readonly>
+ <address type="drive" controller="0" bus="0" target="0" unit="1"></address>
+ </disk>
+ <controller type="scsi" index="0" model="virtio-scsi">
+ <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"></address>
+ </controller>
+ <controller type="pci" model="pci-root"></controller>
+ <serial type="unix">
+ <source mode="connect" path="/var/lib/libvirt/streamer.sock">
+ <reconnect enabled="yes" timeout="1"></reconnect>
+ </source>
+ <target port="0"></target>
+ </serial>
+ <input type="tablet" bus="usb"></input>
+ <graphics type="vnc" port="-1"></graphics>
+ <video>
+ <model type="cirrus"></model>
+ </video>
+ </devices>
+ <commandline xmlns="http://libvirt.org/schemas/domain/qemu/1.0">
+ <env name="VIRTLET_EMULATOR" value="/usr/bin/kvm"></env>
+ <env name="VIRTLET_NET_KEY" value="/tmp/fakenetns"></env>
+ <env name="VIRTLET_CONTAINER_ID" value="231700d5-c9a6-5a49-738d-99a954c51550"></env>
+ <env name="VIRTLET_CONTAINER_LOG_PATH" value="/var/log/pods/69eec606-0493-5825-73a4-c5e0c0236155/container1_42.log"></env>
+ </commandline>
+ </domain>
+- name: 'domain conn: virtlet-231700d5-c9a6-container1: Create'
+- name: 'domain conn: virtlet-231700d5-c9a6-container1: iso image'
+ value:
+ meta-data: '{"instance-id":"testName_0.default","local-hostname":"testName_0"}'
+ network-config: |
+ version: 1
+ user-data: |
+ #cloud-config
+- name: 'domain conn: virtlet-231700d5-c9a6-container1: Destroy'
+- name: 'domain conn: virtlet-231700d5-c9a6-container1: Undefine'
+- name: CMD
+ value:
+ cmd: dmsetup remove virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155

pkg/libvirttools/TestDomainDefinitions__plain_domain.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainDefinitions__raw_block_volume.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainDefinitions__raw_devices.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainDefinitions__vcpu_count.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainDefinitions__virtio_disk_driver.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainDefinitions__volumes.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestDomainForcedShutdown.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
  value: fake/image1
 - name: 'storage: CreateStoragePool'
  value: |-

pkg/libvirttools/TestPersistentRootVolume__first_image_too_big.out.yaml

@@ -0,0 +1,7 @@
+- name: setup
+- name: 'image: GetImagePathDigestAndVirtualSize'
+ value: persistent/image1
+- name: CMD
+ value:
+ cmd: blockdev --getsz /dev/rootdev
+ stdout: "8"