virtlet-ds.yaml

---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: virtlet
  namespace: kube-system
spec:
  template:
    metadata:
      name: virtlet
      labels:
        runtime: virtlet
    spec:
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      # hostPID is true to (1) enable VMs to survive virtlet container restart
      # (to be checked) and (2) to enable the use of nsenter in init container
      hostPID: true
      # bootstrap procedure needs to create a configmap in kube-system namespace
      serviceAccountName: virtlet

      # only run Virtlet pods on the nodes with extraRuntime=virtlet label
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: extraRuntime
                operator: In
                values:
                - virtlet

      initContainers:
      # The init container copies virtlet's flexvolume driver
      # to the default kubelet plugin dir and ensures that
      # the directories needed by libvirt & virtlet exist on the host
      - name: prepare-node
        image: mirantis/virtlet
        imagePullPolicy: IfNotPresent
        command:
        - /prepare-node.sh
        volumeMounts:
        - name: k8s-flexvolume-plugins-dir
          mountPath: /kubelet-volume-plugins
        - name: run
          mountPath: /run
        - name: dockersock
          mountPath: /var/run/docker.sock
        - name: log
          mountPath: /hostlog
        # for ensuring that /var/lib/libvirt/images exists on node
        - name: var-lib
          mountPath: /host-var-lib
        securityContext:
          privileged: true

      containers:
      - name: libvirt
        image: mirantis/virtlet
        # In case we inject local virtlet image we want to use it not officially available one
        imagePullPolicy: IfNotPresent
        command:
        - /libvirt.sh
        volumeMounts:
        - mountPath: /sys/fs/cgroup
          name: cgroup
        - mountPath: /lib/modules
          name: modules
          readOnly: true
        - mountPath: /boot
          name: boot
          readOnly: true
        - mountPath: /run
          name: run
        - mountPath: /var/lib/virtlet
          name: virtlet
        - mountPath: /var/lib/libvirt
          name: libvirt
        - mountPath: /var/run/libvirt
          name: libvirt-sockets
        # the log dir is needed here because otherwise libvirt will produce errors
        # like this:
        # Unable to pre-create chardev file '/var/log/vms/afd75bbb-8e97-11e7-9561-02420ac00002/cirros-vm_0.log': No such file or directory
        - name: vms-log
          mountPath: /var/log/vms
        - name: dev
          mountPath: /dev
        securityContext:
          privileged: true
        env:
        - name: VIRTLET_DISABLE_KVM
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: disable_kvm
              optional: true
      - name: virtlet
        image: mirantis/virtlet
        # In case we inject local virtlet image we want to use it not officially available one
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /run
          name: run
        # /boot and /lib/modules are required by supermin
        - mountPath: /lib/modules
          name: modules
          readOnly: true
        - mountPath: /boot
          name: boot
          readOnly: true
        - mountPath: /var/lib/virtlet
          name: virtlet
        - mountPath: /var/lib/libvirt
          name: libvirt
        - mountPath: /etc/cni
          name: cniconf
        - mountPath: /opt/cni/bin.orig
          name: cnibin
        - mountPath: /var/run/libvirt
          name: libvirt-sockets
        - mountPath: /var/lib/cni
          name: cnidata
        - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
          name: k8s-flexvolume-plugins-dir
          # below `:shared` is unofficial way to pass this option docker
          # which then will allow virtlet to see what kubelet mounts in
          # underlaying directories, after virtlet container is created
        - mountPath: /var/lib/kubelet/pods:shared
          name: k8s-pods-dir
        - name: vms-log
          mountPath: /var/log/vms
        - mountPath: /etc/virtlet/images
          name: image-name-translations
        - name: pods-log
          mountPath: /kubernetes-log
        securityContext:
          privileged: true
        env:
        - name: VIRTLET_DISABLE_KVM
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: disable_kvm
              optional: true
        - name: VIRTLET_DOWNLOAD_PROTOCOL
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: download_protocol
              optional: true
        - name: VIRTLET_LOGLEVEL
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: loglevel
              optional: true
        - name: VIRTLET_CALICO_SUBNET
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: calico-subnet
              optional: true
        - name: IMAGE_REGEXP_TRANSLATION
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: image_regexp_translation
              optional: true
        - name: VIRTLET_DISABLE_LOGGING
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: disable_logging
              optional: true
        - name: IMAGE_TRANSLATIONS_DIR
          value: /etc/virtlet/images
        - name: KUBERNETES_POD_LOGS
          value: "/kubernetes-log"
      - name: vms
        image: mirantis/virtlet
        imagePullPolicy: IfNotPresent
        command:
        - /vms.sh
        volumeMounts:
        - mountPath: /var/lib/virtlet
          name: virtlet
        - mountPath: /var/lib/libvirt
          name: libvirt
        - name: vms-log
          mountPath: /var/log/vms
        - name: dev
          mountPath: /dev
      volumes:
      # /dev is needed for host raw device access
      - hostPath:
          path: /dev
        name: dev
      - hostPath:
          path: /sys/fs/cgroup
        name: cgroup
      - hostPath:
          path: /lib/modules
        name: modules
      - hostPath:
          path: /boot
        name: boot
      - hostPath:
          path: /run
        name: run
      # TODO: don't hardcode docker socket location here
      # This will require CRI proxy installation to run
      # in host mount namespace.
      - hostPath:
          path: /var/run/docker.sock
        name: dockersock
      - hostPath:
          path: /var/lib/virtlet
        name: virtlet
      - hostPath:
          path: /var/lib/libvirt
        name: libvirt
      - hostPath:
          path: /etc/cni
        name: cniconf
      - hostPath:
          path: /opt/cni/bin
        name: cnibin
      - hostPath:
          path: /var/lib/cni
        name: cnidata
      - hostPath:
          path: /var/log
        name: log
      - hostPath:
          path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
        name: k8s-flexvolume-plugins-dir
      - hostPath:
          path: /var/lib/kubelet/pods
        name: k8s-pods-dir
      - hostPath:
          path: /var/lib
        name: var-lib
      - hostPath:
          path: /var/log/virtlet/vms
        name: vms-log
      - hostPath:
          path: /var/run/libvirt
        name: libvirt-sockets
      - hostPath:
          path: /var/log/pods
        name: pods-log
      - configMap:
          name: virtlet-image-translations
        name: image-name-translations
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: virtlet
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: virtlet
subjects:
- kind: ServiceAccount
  name: virtlet
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: virtlet
  namespace: kube-system
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: configmap-reader
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: virtlet-userdata-reader
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  - secrets
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: kubelet-node-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: configmap-reader
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:nodes
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: vm-userdata-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: virtlet-userdata-reader
subjects:
- kind: ServiceAccount
  name: virtlet
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: virtlet-crd
rules:
  - apiGroups:
      - "apiextensions.k8s.io"
    resources:
      - customresourcedefinitions
    verbs:
      - create
  - apiGroups:
      - "virtlet.k8s"
    resources:
      - virtletimagemappings
    verbs:
      - list
      - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: virtlet-crd
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: virtlet-crd
subjects:
- kind: ServiceAccount
  name: virtlet
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: virtlet
  namespace: kube-system