Per-node config

.codeclimate.yml

@@ -50,3 +50,5 @@ exclude_patterns:
  # CRI fields / consts, but make CodeClimate unhappy due to
  # underscores and "Id" instead of "ID"
  - "pkg/metadata/types/types.go"
+ # gofmt issue in the generated code
+ - "pkg/client/clientset/versioned/clientset.go"

build/cmd.sh

@@ -9,6 +9,7 @@ VIRTLET_IMAGE="${VIRTLET_IMAGE:-mirantis/virtlet}"
 VIRTLET_SKIP_RSYNC="${VIRTLET_SKIP_RSYNC:-}"
 VIRTLET_RSYNC_PORT="${VIRTLET_RSYNC_PORT:-18730}"
 VIRTLET_ON_MASTER="${VIRTLET_ON_MASTER:-}"
+VIRTLET_MULTI_NODE="${VIRTLET_MULTI_NODE:-}"
 # XXX: try to extract the docker socket path from DOCKER_HOST if it's set to unix://...
 DOCKER_SOCKET_PATH="${DOCKER_SOCKET_PATH:-/var/run/docker.sock}"
 FORCE_UPDATE_IMAGE="${FORCE_UPDATE_IMAGE:-}"
@@ -32,9 +33,15 @@ exclude=(
 )
 rsync_pw_file="${project_dir}/_output/rsync.password"
 busybox_image=busybox:1.27.2
-virtlet_node=kube-node-1
+virtlet_nodes=()
 if [[ ${VIRTLET_ON_MASTER} ]]; then
- virtlet_node=kube-master
+ virtlet_nodes+=(kube-master)
+fi
+if [[ !${VIRTLET_ON_MASTER} || ${VIRTLET_MULTI_NODE} ]]; then
+ virtlet_nodes+=(kube-node-1)
+fi
+if [[ ${VIRTLET_MULTI_NODE} ]]; then
+ virtlet_nodes+=(kube-node-2)
 fi
 bindata_modtime=1522279343
 bindata_out="pkg/tools/bindata.go"
@@ -169,6 +176,7 @@ function ensure_build_container {
  -e CIRCLE_PULL_REQUEST="${CIRCLE_PULL_REQUEST:-}" \
  -e CIRCLE_BRANCH="${CIRCLE_PULL_REQUEST:-}" \
  -e VIRTLET_ON_MASTER="${VIRTLET_ON_MASTER:-}" \
+ -e VIRTLET_MULTI_NODE="${VIRTLET_MULTI_NODE:-}" \
  -e GITHUB_TOKEN="${GITHUB_TOKEN:-}" \
  ${docker_cert_args[@]+"${docker_cert_args[@]}"} \
  --name virtlet-build \
@@ -244,6 +252,7 @@ function copy_back {
 }
 
 function copy_dind_internal {
+ local virtlet_node="${1}"
  if ! docker volume ls -q | grep -q "^kubeadm-dind-${virtlet_node}$"; then
  echo "No active or snapshotted kubeadm-dind-cluster" >&2
  exit 1
@@ -260,12 +269,14 @@ function kvm_ok {
  # The check is done inside the virtlet node container because it
  # has proper /lib/modules from the docker host. Also, it'll have
  # to use the virtlet image later anyway.
- if ! docker exec "${virtlet_node}" docker run --privileged --rm -v /lib/modules:/lib/modules "${VIRTLET_IMAGE}" kvm-ok; then
+ # Use kube-master node as all of the DIND nodes in the cluster are similar
+ if ! docker exec kube-master docker run --privileged --rm -v /lib/modules:/lib/modules "${VIRTLET_IMAGE}" kvm-ok; then
  return 1
  fi
 }
 
-function start_dind {
+function prepare_node {
+ local virtlet_node="${1}"
  ensure_build_container
  if ! docker exec "${virtlet_node}" dpkg-query -W criproxy-nodeps >&/dev/null; then
  echo >&2 "Installing CRI proxy package the node container..."
@@ -286,11 +297,14 @@ function start_dind {
  echo >&2 "Propagating Virtlet image to the node container..."
  vcmd "docker save '${virtlet_image}' | docker exec -i '${virtlet_node}' docker load"
  fi
+ kubectl label node --overwrite "${virtlet_node}" extraRuntime=virtlet
+}
+
+function start_dind {
  local -a virtlet_config=(--from-literal=image_regexp_translation="${IMAGE_REGEXP_TRANSLATION}")
  if ! kvm_ok || [[ ${VIRTLET_DISABLE_KVM:-} ]]; then
  virtlet_config+=(--from-literal=disable_kvm=y)
  fi
- kubectl label node --overwrite "${virtlet_node}" extraRuntime=virtlet
  kubectl create configmap -n kube-system virtlet-config "${virtlet_config[@]}"
  kubectl create configmap -n kube-system virtlet-image-translations --from-file "${project_dir}/deploy/images.yaml"
  start_virtlet
@@ -474,6 +488,20 @@ function update_bindata_internal {
  go-bindata -modtime "${bindata_modtime}" -o "${bindata_out}" -pkg "${bindata_pkg}" "${bindata_dir}"
 }
 
+function update_docs_internal {
+ if [[ ! -f _output/virtletctl ]]; then
+ echo >&2 "Please run build/cmd.sh build first"
+ fi
+ _output/virtletctl gendoc docs/virtletctl
+ tempfile="$(tempfile)"
+ _output/virtletctl gendoc --config >"${tempfile}"
+ sed -i "/<!-- begin -->/,/<!-- end -->/{
+//!d
+/begin/r ${tempfile}
+}" docs/config.md
+ rm -f "${tempfile}"
+}
+
 function usage {
  echo >&2 "Usage:"
  echo >&2 " $0 build"
@@ -484,6 +512,8 @@ function usage {
  echo >&2 " $0 vsh"
  echo >&2 " $0 stop"
  echo >&2 " $0 clean"
+ echo >&2 " $0 update-bindata"
+ echo >&2 " $0 update-docs"
  echo >&2 " $0 gotest [TEST_ARGS...]"
  echo >&2 " $0 gobuild [BUILD_ARGS...]"
  echo >&2 " $0 run CMD..."
@@ -537,6 +567,14 @@ case "${cmd}" in
  update-bindata-internal)
  update_bindata_internal
  ;;
+ update-docs)
+ vcmd "build/cmd.sh update-docs-internal"
+ rm -rf "${project_dir}/docs/virtletctl"
+ docker exec virtlet-build tar -C "${remote_project_dir}" -c docs/config.md docs/virtletctl | tar -C "${project_dir}" -xv
+ ;;
+ update-docs-internal)
+ update_docs_internal
+ ;;
  run)
  vcmd "$*"
  ;;
@@ -562,9 +600,14 @@ case "${cmd}" in
  e2e "$@"
  ;;
  copy-dind-internal)
- copy_dind_internal
+ for virtlet_node in "${virtlet_nodes[@]}"; do
+ copy_dind_internal "${virtlet_node}"
+ done
  ;;
  start-dind)
+ for virtlet_node in "${virtlet_nodes[@]}"; do
+ prepare_node "${virtlet_node}"
+ done
  start_dind
  ;;
  start-build-container)

build/custom-boilerplate.go.txt

@@ -0,0 +1,15 @@
+/*
+Copyright YEAR Mirantis
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/

build/update-codegen.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# TODO: move this to cmd.sh once we upgrade to Go 1.10
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT="$(dirname ${BASH_SOURCE})/.."
+CODEGEN_PKG="${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)}"
+
+vendor/k8s.io/code-generator/generate-groups.sh all \
+ github.com/Mirantis/virtlet/pkg/client github.com/Mirantis/virtlet/pkg/api \
+ virtlet.k8s:v1 \
+ --go-header-file "${SCRIPT_ROOT}/build/custom-boilerplate.go.txt"
+
+# fix import url case issues
+find "${SCRIPT_ROOT}/pkg/client" -name '*.go' -exec sed -i 's@github\.com/mirantis/virtlet@github\.com/Mirantis/virtlet@g' '{}' \;

cmd/virtlet/virtlet.go

@@ -17,97 +17,80 @@ limitations under the License.
 package main
 
 import (
- "flag"
+ goflag "flag"
+ "fmt"
  "math/rand"
  "os"
  "os/exec"
  "time"
 
  "github.com/golang/glog"
+ flag "github.com/spf13/pflag"
+ "k8s.io/client-go/tools/clientcmd"
 
+ "github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1"
  "github.com/Mirantis/virtlet/pkg/cni"
+ "github.com/Mirantis/virtlet/pkg/config"
  "github.com/Mirantis/virtlet/pkg/libvirttools"
  "github.com/Mirantis/virtlet/pkg/manager"
  "github.com/Mirantis/virtlet/pkg/tapmanager"
  "github.com/Mirantis/virtlet/pkg/utils"
  "github.com/Mirantis/virtlet/pkg/version"
 )
 
+const (
+ wantTapManagerEnv = "WANT_TAP_MANAGER"
+ nodeNameEnv = "KUBE_NODE_NAME"
+)
+
 var (
- libvirtURI = flag.String("libvirt-uri", "qemu:///system",
- "Libvirt connection URI")
- imageDir = flag.String("image dir", "/var/lib/virtlet/images",
- "Image directory")
- boltPath = flag.String("bolt-path", "/var/lib/virtlet/virtlet.db",
- "Path to the bolt database file")
- listen = flag.String("listen", "/run/virtlet.sock",
- "The unix socket to listen on, e.g. /run/virtlet.sock")
- cniPluginsDir = flag.String("cni-bin-dir", "/opt/cni/bin",
- "Path to CNI plugin binaries")
- cniConfigsDir = flag.String("cni-conf-dir", "/etc/cni/net.d",
- "Location of CNI configurations (first file name in lexicographic order will be chosen)")
- imageDownloadProtocol = flag.String("image-download-protocol", "https",
- "Image download protocol. Can be https (default) or http.")
- rawDevices = flag.String("raw-devices", "loop*",
- "Comma separated list of raw device glob patterns to which VM can have an access (with skipped /dev/ prefix)")
- fdServerSocketPath = flag.String("fd-server-socket-path", "/var/lib/virtlet/tapfdserver.sock",
- "Path to fd server socket")
- imageTranslationConfigsDir = flag.String("image-translations-dir", "",
- "Image name translation configs directory")
+ dumpConfig = flag.Bool("dump-config", false, "Dump node-specific Virtlet config as a shell script and exit")
  displayVersion = flag.Bool("version", false, "Display version and exit")
  versionFormat = flag.String("version-format", "text", "Version format to use (text, short, json, yaml)")
 )
 
-const (
- WantTapManagerEnv = "WANT_TAP_MANAGER"
-)
+func configWithDefaults(cfg *v1.VirtletConfig) *v1.VirtletConfig {
+ r := config.GetDefaultConfig()
+ config.Override(r, cfg)
+ return r
+}
 
-func runVirtlet() {
- manager := manager.NewVirtletManager(&manager.VirtletConfig{
- FDServerSocketPath: *fdServerSocketPath,
- DatabasePath: *boltPath,
- DownloadProtocol: *imageDownloadProtocol,
- ImageDir: *imageDir,
- ImageTranslationConfigsDir: *imageTranslationConfigsDir,
- LibvirtURI: *libvirtURI,
- RawDevices: *rawDevices,
- CRISocketPath: *listen,
- DisableLogging: os.Getenv("VIRTLET_DISABLE_LOGGING") != "",
- })
+func runVirtlet(config *v1.VirtletConfig, clientCfg clientcmd.ClientConfig) {
+ manager := manager.NewVirtletManager(config, nil, clientCfg)
  if err := manager.Run(); err != nil {
  glog.Errorf("Error: %v", err)
  os.Exit(1)
  }
 }
 
-func runTapManager() {
- cniClient, err := cni.NewClient(*cniPluginsDir, *cniConfigsDir)
+func runTapManager(config *v1.VirtletConfig) {
+ cniClient, err := cni.NewClient(*config.CNIPluginDir, *config.CNIConfigDir)
  if err != nil {
  glog.Errorf("Error initializing CNI client: %v", err)
  os.Exit(1)
  }
- src, err := tapmanager.NewTapFDSource(cniClient)
+ src, err := tapmanager.NewTapFDSource(cniClient, *config.EnableSriov, *config.CalicoSubnetSize)
  if err != nil {
  glog.Errorf("Error creating tap fd source: %v", err)
  os.Exit(1)
  }
- os.Remove(*fdServerSocketPath) // FIXME
- s := tapmanager.NewFDServer(*fdServerSocketPath, src)
+ os.Remove(*config.FDServerSocketPath) // FIXME
+ s := tapmanager.NewFDServer(*config.FDServerSocketPath, src)
  if err = s.Serve(); err != nil {
  glog.Errorf("FD server returned error: %v", err)
  os.Exit(1)
  }
- if err := libvirttools.ChownForEmulator(*fdServerSocketPath); err != nil {
+ if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath); err != nil {
  glog.Warningf("Couldn't set tapmanager socket permissions: %v", err)
  }
  for {
  time.Sleep(1000 * time.Hour)
  }
 }
 
-func startTapManagerProcess() {
+func startTapManagerProcess(config *v1.VirtletConfig) {
  cmd := exec.Command(os.Args[0], os.Args[1:]...)
- cmd.Env = append(os.Environ(), WantTapManagerEnv+"=1")
+ cmd.Env = append(os.Environ(), wantTapManagerEnv+"=1")
  cmd.Stdout = os.Stdout
  cmd.Stderr = os.Stderr
  // Here we make this process die with the main Virtlet process.
@@ -131,19 +114,44 @@ func printVersion() {
  }
 }
 
+func setLogLevel(config *v1.VirtletConfig) {
+ goflag.CommandLine.Parse([]string{
+ fmt.Sprintf("-v=%d", config.LogLevel),
+ "-logtostderr=true",
+ })
+}
+
 func main() {
  utils.HandleNsFixReexec()
+ clientCfg := utils.BindFlags(flag.CommandLine)
+ var cb *config.Binder
+ cb = config.NewBinder(flag.CommandLine)
  flag.Parse()
- if *displayVersion {
- printVersion()
- os.Exit(0)
- }
+ localConfig := cb.GetConfig()
 
  rand.Seed(time.Now().UnixNano())
- if os.Getenv(WantTapManagerEnv) == "" {
- startTapManagerProcess()
- runVirtlet()
- } else {
- runTapManager()
+ setLogLevel(configWithDefaults(localConfig))
+ switch {
+ case os.Getenv(wantTapManagerEnv) != "":
+ localConfig = configWithDefaults(localConfig)
+ runTapManager(localConfig)
+ case *displayVersion:
+ printVersion()
+ case *dumpConfig:
+ nodeConfig := config.NewNodeConfig(clientCfg)
+ nodeName := os.Getenv(nodeNameEnv)
+ cfg, err := nodeConfig.LoadConfig(localConfig, nodeName)
+ if err != nil {
+ glog.Warningf("Failed to load per-node configs, using local config only: %v", err)
+ cfg = localConfig
+ }
+ if _, err := os.Stdout.Write([]byte(config.DumpEnv(cfg))); err != nil {
+ glog.Errorf("Error writing config: %v", err)
+ os.Exit(1)
+ }
+ default:
+ localConfig = configWithDefaults(localConfig)
+ startTapManagerProcess(localConfig)
+ runVirtlet(localConfig, clientCfg)
  }
 }

cmd/virtletctl/virtletctl.go

@@ -48,7 +48,7 @@ func newRootCmd() *cobra.Command {
  cmd.AddCommand(tools.NewSSHCmd(client, os.Stdout, ""))
  cmd.AddCommand(tools.NewVNCCmd(client, os.Stdout, true))
  cmd.AddCommand(tools.NewInstallCmd(cmd, "", ""))
- cmd.AddCommand(tools.NewGenDocCmd(cmd))
+ cmd.AddCommand(tools.NewGenDocCmd(cmd, os.Stdout))
  cmd.AddCommand(tools.NewGenCmd(os.Stdout))
  cmd.AddCommand(tools.NewVersionCommand(client, os.Stdout, nil))
 

deploy/README.md

@@ -75,6 +75,9 @@ by Virtlet when it's deployed using k8s yaml produced by `virtletctl gen`:
  * `image_regexp_translation` - enables regexp syntax for the image name translation rules.
  * `disable_logging` - disables log streaming from VMs. Use "1" to disable.
 
+It's also possible to set [per-node configuration](../docs/config.md)
+for Virtlet using CRDs.
+
 ## Removing Virtlet
 
 In order to remove Virtlet, first you need to delete all the VM pods.