Merge pull request #411 from Mirantis/ivan4th/calico

Calico support

.circleci/config.yml

@@ -113,6 +113,43 @@ push_images: &push_images
  docker push "${img}"
  done
 
+e2e: &e2e
+ <<: *defaults
+ steps:
+ - run:
+ <<: *prereqs
+ - checkout
+ - setup_remote_docker
+ - run:
+ <<: *setup_env
+ - attach_workspace:
+ at: _output
+ - run:
+ name: Restore virtlet image
+ command: |
+ docker load -i _output/virtlet.tar
+ - run:
+ name: Start the demo
+ command: |
+ build/portforward.sh 8080&
+ if [[ ${CIRCLE_JOB} = e2e_calico ]]; then
+ export CNI_PLUGIN=calico
+ echo >&2 "*** Using Calico CNI"
+ fi
+ VIRTLET_DEMO_RELEASE=master \
+ SKIP_SNAPSHOT=1 \
+ NONINTERACTIVE=1 \
+ NO_VM_CONSOLE=1 \
+ INJECT_LOCAL_IMAGE=1 \
+ VIRTLET_DEMO_RELEASE=master \
+ BASE_LOCATION="$PWD" \
+ deploy/demo.sh
+ - run:
+ name: Run e2e tests
+ command: |
+ build/portforward.sh 8080&
+ _output/virtlet-e2e-tests -test.v
+
 version: 2
 jobs:
  prepare_build:
@@ -243,37 +280,10 @@ jobs:
  build/cmd.sh integration
 
  e2e:
- <<: *defaults
- steps:
- - run:
- <<: *prereqs
- - checkout
- - setup_remote_docker
- - run:
- <<: *setup_env
- - attach_workspace:
- at: _output
- - run:
- name: Restore virtlet image
- command: |
- docker load -i _output/virtlet.tar
- - run:
- name: Start the demo
- command: |
- build/portforward.sh 8080&
- VIRTLET_DEMO_RELEASE=master \
- SKIP_SNAPSHOT=1 \
- NONINTERACTIVE=1 \
- NO_VM_CONSOLE=1 \
- INJECT_LOCAL_IMAGE=1 \
- VIRTLET_DEMO_RELEASE=master \
- BASE_LOCATION="$PWD" \
- deploy/demo.sh
- - run:
- name: Run e2e tests
- command: |
- build/portforward.sh 8080&
- _output/virtlet-e2e-tests -test.v
+ <<: *e2e
+
+ e2e_calico:
+ <<: *e2e
 
  push_branch:
  <<: *push_images
@@ -318,6 +328,13 @@ workflows:
  tags:
  only:
  - /^v[0-9].*/
+ - e2e_calico:
+ requires:
+ - build
+ filters:
+ tags:
+ only:
+ - /^v[0-9].*/
  - push_branch:
  requires:
  - build
@@ -329,6 +346,7 @@ workflows:
  requires:
  - test
  - e2e
+ - e2e_calico
  - integration
  filters:
  branches:

README.md

@@ -39,7 +39,7 @@ The demo will start a test cluster, deploy Virtlet on it and then boot a [CirrOS
 examples/vmssh.sh cirros@cirros-vm [command...]
 ```
 
-By default, CNI bridge plugin is used for cluster networking. It's also possible to override this with `flannel` or `weave` plugin, e.g.:
+By default, CNI bridge plugin is used for cluster networking. It's also possible to override this with `calico`, `flannel` or `weave` plugin, e.g.:
 ```
 CNI_PLUGIN=flannel ./demo.sh
 ```

deploy/virtlet-ds-dev.yaml

@@ -164,6 +164,12 @@ spec:
  name: virtlet-config
  key: loglevel
  optional: true
+ - name: VIRTLET_CALICO_SUBNET
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: calico-subnet
+ optional: true
  - name: IMAGE_REGEXP_TRANSLATION
  valueFrom:
  configMapKeyRef:

deploy/virtlet-ds.yaml

@@ -164,6 +164,12 @@ spec:
  name: virtlet-config
  key: loglevel
  optional: true
+ - name: VIRTLET_CALICO_SUBNET
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: calico-subnet
+ optional: true
  - name: IMAGE_REGEXP_TRANSLATION
  valueFrom:
  configMapKeyRef:

docs/networking.md

@@ -81,5 +81,14 @@ socket makes it possible to have all the network related code outside
 `vmwrapper` and have `vmwrapper` just `exec` the emulator instead of
 spawning it as a child process.
 
+[Calico](https://www.projectcalico.org/) CNI plugin needs special treatment
+as it tries to pass a routing configuration that cannot be passed
+over DHCP. For it to work Virtlet patches Calico-provided CNI result,
+replacing Calico's unreachable fake gateway with another fake gateway
+with an IP address acquired from Calico IPAM. A proper node subnet must
+be set for Calico-based virtlet installations. It's controlled by
+`calico-subnet` key Virtlet configmap (denoting the number of 1s in
+the netmask) and defaults to `24`.
+
 **NOTE:** Virtlet doesn't support `hostNetwork` pod setting because it
 cannot be impelemnted for VM in a meaningful way.

examples/ubuntu-vm.yaml

@@ -6,6 +6,14 @@ metadata:
  kubernetes.io/target-runtime: virtlet
  VirtletCloudInitUserData: |
  ssh_pwauth: True
+ users:
+ - name: testuser
+ gecos: User
+ primary-group: testuser
+ groups: users
+ lock_passwd: false
+ passwd: "$6$rounds=4096$wPs4Hz4tfs$a8ssMnlvH.3GX88yxXKF2cKMlVULsnydoOKgkuStTErTq2dzKZiIx9R/pPWWh5JLxzoZEx7lsSX5T2jW5WISi1"
+ sudo: ALL=(ALL) NOPASSWD:ALL
  VirtletSSHKeys: |
  ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
 spec:

pkg/cni/client.go

@@ -23,6 +23,8 @@ import (
  cnicurrent "github.com/containernetworking/cni/pkg/types/current"
  "github.com/davecgh/go-spew/spew"
  "github.com/golang/glog"
+
+ "github.com/Mirantis/virtlet/pkg/utils"
 )
 
 type Client struct {
@@ -32,6 +34,7 @@ type Client struct {
 
 func NewClient(pluginsDir, configsDir string) (*Client, error) {
  configuration, err := ReadConfiguration(configsDir)
+ glog.V(3).Infof("CNI config: name: %q type: %q", configuration.Network.Name, configuration.Network.Type)
  if err != nil {
  return nil, fmt.Errorf("failed to read CNI configuration: %v", err)
  }
@@ -42,18 +45,37 @@ func NewClient(pluginsDir, configsDir string) (*Client, error) {
  }, nil
 }
 
+func (c *Client) Type() string { return c.configuration.Network.Type }
+
 func (c *Client) cniRuntimeConf(podId, podName, podNs string) *libcni.RuntimeConf {
- return &libcni.RuntimeConf{
+ r := &libcni.RuntimeConf{
  ContainerID: podId,
  NetNS: PodNetNSPath(podId),
  IfName: "virtlet-eth0",
- Args: [][2]string{
+ }
+ if podName != "" && podNs != "" {
+ r.Args = [][2]string{
  {"IgnoreUnknown", "1"},
  {"K8S_POD_NAMESPACE", podNs},
  {"K8S_POD_NAME", podName},
  {"K8S_POD_INFRA_CONTAINER_ID", podId},
- },
+ }
+ }
+ return r
+}
+
+// GetDummyNetwork creates a dummy network using CNI plugin.
+// It's used for making a dummy gateway for Calico CNI plugin.
+func (c *Client) GetDummyNetwork() (*cnicurrent.Result, error) {
+ // TODO: virtlet pod restarts should not grab another address for
+ // the gateway. That's not a big problem usually though
+ // as the IPs are not returned to Calico so both old
+ // IPs on existing VMs and new ones should work.
+ podId := utils.NewUuid()
+ if err := CreateNetNS(podId); err != nil {
+ return nil, fmt.Errorf("couldn't create netns for fake pod %q: %v", podId, err)
  }
+ return c.AddSandboxToNetwork(podId, "", "")
 }
 
 func (c *Client) AddSandboxToNetwork(podId, podName, podNs string) (*cnicurrent.Result, error) {

pkg/tapmanager/tapfdsource.go

@@ -18,7 +18,11 @@ package tapmanager
 
 import (
  "encoding/json"
+ "errors"
  "fmt"
+ "net"
+ "os"
+ "strconv"
  "sync"
  "time"
 
@@ -33,6 +37,12 @@ import (
  "github.com/Mirantis/virtlet/pkg/nettools"
 )
 
+const (
+ calicoNetType = "calico"
+ calicoDefaultSubnet = 24
+ calicoSubnetVar = "VIRTLET_CALICO_SUBNET"
+)
+
 // PodNetworkDesc contains the data that are required by TapFDSource
 // to set up a tap device for a VM
 type PodNetworkDesc struct {
@@ -70,8 +80,9 @@ type podNetwork struct {
 type TapFDSource struct {
  sync.Mutex
 
- cniClient *cni.Client
- fdMap map[string]*podNetwork
+ cniClient *cni.Client
+ dummyGateway net.IP
+ fdMap map[string]*podNetwork
 }
 
 var _ FDSource = &TapFDSource{}
@@ -84,10 +95,28 @@ func NewTapFDSource(cniPluginsDir, cniConfigsDir string) (*TapFDSource, error) {
  return nil, err
  }
 
- return &TapFDSource{
+ s := &TapFDSource{
  cniClient: cniClient,
  fdMap: make(map[string]*podNetwork),
- }, nil
+ }
+
+ // Calico needs special treatment here.
+ // We need to make network config DHCP-compatible by throwing away
+ // Calico's gateway and dev route and using a fake gateway instead.
+ // The fake gateway is just an IP address allocated by Calico IPAM,
+ // it's needed for proper ARP resppnses for VMs.
+ if cniClient.Type() == calicoNetType {
+ dummyResult, err := cniClient.GetDummyNetwork()
+ if err != nil {
+ return nil, err
+ }
+ if len(dummyResult.IPs) != 1 {
+ return nil, fmt.Errorf("expected 1 ip for the dummy network, but got %d", len(dummyResult.IPs))
+ }
+ s.dummyGateway = dummyResult.IPs[0].Address.IP
+ }
+
+ return s, nil
 }
 
 // GetFD implements GetFD method of FDSource interface
@@ -121,6 +150,27 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) {
 
  netConfig := payload.CNIConfig
 
+ // Calico needs network config to be adjusted for DHCP compatibility
+ if s.dummyGateway != nil {
+ if len(netConfig.IPs) != 1 {
+ return 0, nil, errors.New("didn't expect more than one IP config")
+ }
+ if netConfig.IPs[0].Version != "4" {
+ return 0, nil, errors.New("IPv4 config was expected")
+ }
+ netConfig.IPs[0].Address.Mask = netmaskForCalico()
+ netConfig.IPs[0].Gateway = s.dummyGateway
+ netConfig.Routes = []*cnitypes.Route{
+ {
+ Dst: net.IPNet{
+ IP: net.IP{0, 0, 0, 0},
+ Mask: net.IPMask{0, 0, 0, 0},
+ },
+ GW: s.dummyGateway,
+ },
+ }
+ }
+
  netNSPath := cni.PodNetNSPath(pnd.PodId)
  vmNS, err := ns.GetNS(netNSPath)
  if err != nil {
@@ -143,8 +193,10 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) {
 
  // NOTE: older CNI plugins don't include the hardware address
  // in Result, but it's needed for Cloud-Init based
- // network setup, so we add it here if it's missing
- ensureCNIInterfaceHwAddress(netConfig, csn)
+ // network setup, so we add it here if it's missing.
+ // Also, some of the plugins may skip adding routes
+ // to the CNI result, so we must add them, too
+ fixCNIResult(netConfig, csn)
 
  // TODO: now CNIConfig should always contain interface mac address, so there
  // is no reason to pass it as separate field in dhcp.Config,
@@ -244,7 +296,7 @@ func (s *TapFDSource) GetInfo(key string) ([]byte, error) {
  return pn.csn.HardwareAddr, nil
 }
 
-func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) {
+func fixCNIResult(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) {
  // If there's no interface info in netConfig, we can assume that we're dealing
  // with an old-style CNI plugin which only supports a single network interface
  if len(netConfig.Interfaces) > 0 {
@@ -260,4 +312,21 @@ func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.Con
  for _, IP := range netConfig.IPs {
  IP.Interface = 0
  }
+
+ if len(netConfig.Routes) == 0 {
+ netConfig.Routes = csn.Result.Routes
+ }
+}
+
+func netmaskForCalico() net.IPMask {
+ n := calicoDefaultSubnet
+ subnetStr := os.Getenv(calicoSubnetVar)
+ if subnetStr != "" {
+ n, err := strconv.Atoi(subnetStr)
+ if err != nil || n <= 0 || n > 30 {
+ glog.Warningf("bad calico subnet %q, using /%d", subnetStr, calicoDefaultSubnet)
+ n = calicoDefaultSubnet
+ }
+ }
+ return net.CIDRMask(n, 32)
 }