Implement per-node config

Also, use k8s.io/code-generator for CRDs.

build/custom-boilerplate.go.txt

@@ -0,0 +1,15 @@
+/*
+Copyright YEAR Mirantis
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/

build/update-codegen.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# TODO: move this to cmd.sh once we upgrade to Go 1.10
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT="$(dirname ${BASH_SOURCE})/.."
+CODEGEN_PKG="${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)}"
+
+vendor/k8s.io/code-generator/generate-groups.sh all \
+ github.com/Mirantis/virtlet/pkg/client github.com/Mirantis/virtlet/pkg/api \
+ virtlet.k8s:v1 \
+ --go-header-file "${SCRIPT_ROOT}/build/custom-boilerplate.go.txt"
+
+# fix import url case issues
+find "${SCRIPT_ROOT}/pkg/client" -name '*.go' -exec sed -i 's@github\.com/mirantis/virtlet@github\.com/Mirantis/virtlet@g' '{}' \;

cmd/virtlet/virtlet.go

@@ -17,97 +17,77 @@ limitations under the License.
 package main
 
 import (
- "flag"
  "math/rand"
  "os"
  "os/exec"
  "time"
 
  "github.com/golang/glog"
+ flag "github.com/spf13/pflag"
 
+ "github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1"
  "github.com/Mirantis/virtlet/pkg/cni"
+ "github.com/Mirantis/virtlet/pkg/config"
  "github.com/Mirantis/virtlet/pkg/libvirttools"
  "github.com/Mirantis/virtlet/pkg/manager"
  "github.com/Mirantis/virtlet/pkg/tapmanager"
  "github.com/Mirantis/virtlet/pkg/utils"
  "github.com/Mirantis/virtlet/pkg/version"
 )
 
+const (
+ wantTapManagerEnv = "WANT_TAP_MANAGER"
+ nodeNameEnv = "KUBE_NODE_NAME"
+)
+
 var (
- libvirtURI = flag.String("libvirt-uri", "qemu:///system",
- "Libvirt connection URI")
- imageDir = flag.String("image dir", "/var/lib/virtlet/images",
- "Image directory")
- boltPath = flag.String("bolt-path", "/var/lib/virtlet/virtlet.db",
- "Path to the bolt database file")
- listen = flag.String("listen", "/run/virtlet.sock",
- "The unix socket to listen on, e.g. /run/virtlet.sock")
- cniPluginsDir = flag.String("cni-bin-dir", "/opt/cni/bin",
- "Path to CNI plugin binaries")
- cniConfigsDir = flag.String("cni-conf-dir", "/etc/cni/net.d",
- "Location of CNI configurations (first file name in lexicographic order will be chosen)")
- imageDownloadProtocol = flag.String("image-download-protocol", "https",
- "Image download protocol. Can be https (default) or http.")
- rawDevices = flag.String("raw-devices", "loop*",
- "Comma separated list of raw device glob patterns to which VM can have an access (with skipped /dev/ prefix)")
- fdServerSocketPath = flag.String("fd-server-socket-path", "/var/lib/virtlet/tapfdserver.sock",
- "Path to fd server socket")
- imageTranslationConfigsDir = flag.String("image-translations-dir", "",
- "Image name translation configs directory")
+ dumpConfig = flag.Bool("dump-config", false, "Dump node-specific Virtlet config as a shell script and exit")
  displayVersion = flag.Bool("version", false, "Display version and exit")
  versionFormat = flag.String("version-format", "text", "Version format to use (text, short, json, yaml)")
 )
 
-const (
- WantTapManagerEnv = "WANT_TAP_MANAGER"
-)
+func configWithDefaults(cfg *v1.VirtletConfig) *v1.VirtletConfig {
+ r := config.GetDefaultConfig()
+ config.Override(r, cfg)
+ return r
+}
 
-func runVirtlet() {
- manager := manager.NewVirtletManager(&manager.VirtletConfig{
- FDServerSocketPath: *fdServerSocketPath,
- DatabasePath: *boltPath,
- DownloadProtocol: *imageDownloadProtocol,
- ImageDir: *imageDir,
- ImageTranslationConfigsDir: *imageTranslationConfigsDir,
- LibvirtURI: *libvirtURI,
- RawDevices: *rawDevices,
- CRISocketPath: *listen,
- DisableLogging: os.Getenv("VIRTLET_DISABLE_LOGGING") != "",
- })
+func runVirtlet(config *v1.VirtletConfig) {
+ manager := manager.NewVirtletManager(config, nil)
  if err := manager.Run(); err != nil {
  glog.Errorf("Error: %v", err)
  os.Exit(1)
  }
 }
 
-func runTapManager() {
- cniClient, err := cni.NewClient(*cniPluginsDir, *cniConfigsDir)
+func runTapManager(config *v1.VirtletConfig) {
+ cniClient, err := cni.NewClient(*config.CNIPluginDir, *config.CNIConfigDir)
  if err != nil {
  glog.Errorf("Error initializing CNI client: %v", err)
  os.Exit(1)
  }
- src, err := tapmanager.NewTapFDSource(cniClient)
+ src, err := tapmanager.NewTapFDSource(cniClient, *config.EnableSriov, *config.CalicoSubnetSize)
  if err != nil {
  glog.Errorf("Error creating tap fd source: %v", err)
  os.Exit(1)
  }
- os.Remove(*fdServerSocketPath) // FIXME
- s := tapmanager.NewFDServer(*fdServerSocketPath, src)
+ os.Remove(*config.FDServerSocketPath) // FIXME
+ s := tapmanager.NewFDServer(*config.FDServerSocketPath, src)
  if err = s.Serve(); err != nil {
  glog.Errorf("FD server returned error: %v", err)
  os.Exit(1)
  }
- if err := libvirttools.ChownForEmulator(*fdServerSocketPath); err != nil {
+ if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath); err != nil {
  glog.Warningf("Couldn't set tapmanager socket permissions: %v", err)
  }
  for {
  time.Sleep(1000 * time.Hour)
  }
 }
 
-func startTapManagerProcess() {
+func startTapManagerProcess(config *v1.VirtletConfig) {
  cmd := exec.Command(os.Args[0], os.Args[1:]...)
- cmd.Env = append(os.Environ(), WantTapManagerEnv+"=1")
+ cmd.Env = append(os.Environ(), wantTapManagerEnv+"=1")
  cmd.Stdout = os.Stdout
  cmd.Stderr = os.Stderr
  // Here we make this process die with the main Virtlet process.
@@ -133,17 +113,34 @@ func printVersion() {
 
 func main() {
  utils.HandleNsFixReexec()
+ clientCfg := utils.BindFlags(flag.CommandLine)
+ var cb *config.ConfigBinder
+ cb = config.NewConfigBinder(flag.CommandLine)
  flag.Parse()
- if *displayVersion {
- printVersion()
- os.Exit(0)
- }
+ localConfig := cb.GetConfig()
 
  rand.Seed(time.Now().UnixNano())
- if os.Getenv(WantTapManagerEnv) == "" {
- startTapManagerProcess()
- runVirtlet()
- } else {
- runTapManager()
+ switch {
+ case os.Getenv(wantTapManagerEnv) != "":
+ localConfig = configWithDefaults(localConfig)
+ runTapManager(localConfig)
+ case *displayVersion:
+ printVersion()
+ case *dumpConfig:
+ nodeConfig := config.NewNodeConfig(clientCfg)
+ nodeName := os.Getenv(nodeNameEnv)
+ cfg, err := nodeConfig.LoadConfig(localConfig, nodeName)
+ if err != nil {
+ glog.Warning("Failed to load per-node configs, using local config only: %v", err)
+ cfg = localConfig
+ }
+ if _, err := os.Stdout.Write([]byte(config.DumpEnv(cfg))); err != nil {
+ glog.Errorf("Error writing config: %v", err)
+ os.Exit(1)
+ }
+ default:
+ localConfig = configWithDefaults(localConfig)
+ startTapManagerProcess(localConfig)
+ runVirtlet(localConfig)
  }
 }

deploy/data/virtlet-ds.yaml

@@ -54,15 +54,66 @@ spec:
  mountPath: /host-var-lib
  - name: dev
  mountPath: /dev
+ - mountPath: /var/lib/virtlet
+ name: virtlet
  securityContext:
  privileged: true
  env:
+ - name: KUBE_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
  - name: VIRTLET_DISABLE_KVM
  valueFrom:
  configMapKeyRef:
  name: virtlet-config
  key: disable_kvm
  optional: true
+ - name: VIRTLET_SRIOV_SUPPORT
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: sriov_support
+ optional: true
+ - name: VIRTLET_DOWNLOAD_PROTOCOL
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: download_protocol
+ optional: true
+ - name: VIRTLET_LOGLEVEL
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: loglevel
+ optional: true
+ - name: VIRTLET_CALICO_SUBNET
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: calico-subnet
+ optional: true
+ - name: IMAGE_REGEXP_TRANSLATION
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: image_regexp_translation
+ optional: true
+ - name: VIRTLET_RAW_DEVICES
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: raw_devices
+ optional: true
+ - name: VIRTLET_DISABLE_LOGGING
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: disable_logging
+ optional: true
+ - name: VIRTLET_IMAGE_TRANSLATIONS_DIR
+ value: /etc/virtlet/images
 
  containers:
  - name: libvirt
@@ -100,19 +151,6 @@ spec:
  mountPath: /dev
  securityContext:
  privileged: true
- env:
- - name: VIRTLET_SRIOV_SUPPORT
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: sriov_support
- optional: true
- - name: VIRTLET_DISABLE_KVM
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: disable_kvm
- optional: true
  readinessProbe:
  exec:
  command:
@@ -153,57 +191,6 @@ spec:
  mountPath: /var/log/pods
  securityContext:
  privileged: true
- env:
- - name: VIRTLET_DISABLE_KVM
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: disable_kvm
- optional: true
- - name: VIRTLET_DOWNLOAD_PROTOCOL
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: download_protocol
- optional: true
- - name: VIRTLET_LOGLEVEL
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: loglevel
- optional: true
- - name: VIRTLET_CALICO_SUBNET
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: calico-subnet
- optional: true
- - name: IMAGE_REGEXP_TRANSLATION
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: image_regexp_translation
- optional: true
- - name: VIRTLET_DISABLE_LOGGING
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: disable_logging
- optional: true
- - name: VIRTLET_SRIOV_SUPPORT
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: sriov_support
- optional: true
- - name: VIRTLET_RAW_DEVICES
- valueFrom:
- configMapKeyRef:
- name: virtlet-config
- key: raw_devices
- optional: true
- - name: IMAGE_TRANSLATIONS_DIR
- value: /etc/virtlet/images
  readinessProbe:
  exec:
  command: