diff --git a/.circleci/config.yml b/.circleci/config.yml index 74ddc0df3..461413230 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -113,6 +113,43 @@ push_images: &push_images docker push "${img}" done +e2e: &e2e + <<: *defaults + steps: + - run: + <<: *prereqs + - checkout + - setup_remote_docker + - run: + <<: *setup_env + - attach_workspace: + at: _output + - run: + name: Restore virtlet image + command: | + docker load -i _output/virtlet.tar + - run: + name: Start the demo + command: | + build/portforward.sh 8080& + if [[ ${CIRCLE_JOB} = e2e_calico ]]; then + export CNI_PLUGIN=calico + echo >&2 "*** Using Calico CNI" + fi + VIRTLET_DEMO_RELEASE=master \ + SKIP_SNAPSHOT=1 \ + NONINTERACTIVE=1 \ + NO_VM_CONSOLE=1 \ + INJECT_LOCAL_IMAGE=1 \ + VIRTLET_DEMO_RELEASE=master \ + BASE_LOCATION="$PWD" \ + deploy/demo.sh + - run: + name: Run e2e tests + command: | + build/portforward.sh 8080& + _output/virtlet-e2e-tests -test.v + version: 2 jobs: prepare_build: @@ -243,37 +280,10 @@ jobs: build/cmd.sh integration e2e: - <<: *defaults - steps: - - run: - <<: *prereqs - - checkout - - setup_remote_docker - - run: - <<: *setup_env - - attach_workspace: - at: _output - - run: - name: Restore virtlet image - command: | - docker load -i _output/virtlet.tar - - run: - name: Start the demo - command: | - build/portforward.sh 8080& - VIRTLET_DEMO_RELEASE=master \ - SKIP_SNAPSHOT=1 \ - NONINTERACTIVE=1 \ - NO_VM_CONSOLE=1 \ - INJECT_LOCAL_IMAGE=1 \ - VIRTLET_DEMO_RELEASE=master \ - BASE_LOCATION="$PWD" \ - deploy/demo.sh - - run: - name: Run e2e tests - command: | - build/portforward.sh 8080& - _output/virtlet-e2e-tests -test.v + <<: *e2e + + e2e_calico: + <<: *e2e push_branch: <<: *push_images @@ -318,6 +328,13 @@ workflows: tags: only: - /^v[0-9].*/ + - e2e_calico: + requires: + - build + filters: + tags: + only: + - /^v[0-9].*/ - push_branch: requires: - build @@ -329,6 +346,7 @@ workflows: requires: - test - e2e + - e2e_calico - integration filters: branches: diff --git a/README.md b/README.md index 82945ccc8..f54f913f1 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ The demo will start a test cluster, deploy Virtlet on it and then boot a [CirrOS examples/vmssh.sh cirros@cirros-vm [command...] ``` -By default, CNI bridge plugin is used for cluster networking. It's also possible to override this with `flannel` or `weave` plugin, e.g.: +By default, CNI bridge plugin is used for cluster networking. It's also possible to override this with `calico`, `flannel` or `weave` plugin, e.g.: ``` CNI_PLUGIN=flannel ./demo.sh ``` diff --git a/deploy/virtlet-ds-dev.yaml b/deploy/virtlet-ds-dev.yaml index 6c15f88f2..5f292780d 100644 --- a/deploy/virtlet-ds-dev.yaml +++ b/deploy/virtlet-ds-dev.yaml @@ -164,6 +164,12 @@ spec: name: virtlet-config key: loglevel optional: true + - name: VIRTLET_CALICO_SUBNET + valueFrom: + configMapKeyRef: + name: virtlet-config + key: calico-subnet + optional: true - name: IMAGE_REGEXP_TRANSLATION valueFrom: configMapKeyRef: diff --git a/deploy/virtlet-ds.yaml b/deploy/virtlet-ds.yaml index c4f5ef128..b62a231e9 100644 --- a/deploy/virtlet-ds.yaml +++ b/deploy/virtlet-ds.yaml @@ -164,6 +164,12 @@ spec: name: virtlet-config key: loglevel optional: true + - name: VIRTLET_CALICO_SUBNET + valueFrom: + configMapKeyRef: + name: virtlet-config + key: calico-subnet + optional: true - name: IMAGE_REGEXP_TRANSLATION valueFrom: configMapKeyRef: diff --git a/docs/networking.md b/docs/networking.md index ce0188ca5..59d25a24c 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -81,5 +81,14 @@ socket makes it possible to have all the network related code outside `vmwrapper` and have `vmwrapper` just `exec` the emulator instead of spawning it as a child process. +[Calico](https://www.projectcalico.org/) CNI plugin needs special treatment +as it tries to pass a routing configuration that cannot be passed +over DHCP. For it to work Virtlet patches Calico-provided CNI result, +replacing Calico's unreachable fake gateway with another fake gateway +with an IP address acquired from Calico IPAM. A proper node subnet must +be set for Calico-based virtlet installations. It's controlled by +`calico-subnet` key Virtlet configmap (denoting the number of 1s in +the netmask) and defaults to `24`. + **NOTE:** Virtlet doesn't support `hostNetwork` pod setting because it cannot be impelemnted for VM in a meaningful way. diff --git a/examples/ubuntu-vm.yaml b/examples/ubuntu-vm.yaml index 4495a673c..9acd1987d 100644 --- a/examples/ubuntu-vm.yaml +++ b/examples/ubuntu-vm.yaml @@ -6,6 +6,14 @@ metadata: kubernetes.io/target-runtime: virtlet VirtletCloudInitUserData: | ssh_pwauth: True + users: + - name: testuser + gecos: User + primary-group: testuser + groups: users + lock_passwd: false + passwd: "$6$rounds=4096$wPs4Hz4tfs$a8ssMnlvH.3GX88yxXKF2cKMlVULsnydoOKgkuStTErTq2dzKZiIx9R/pPWWh5JLxzoZEx7lsSX5T2jW5WISi1" + sudo: ALL=(ALL) NOPASSWD:ALL VirtletSSHKeys: | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost spec: diff --git a/pkg/cni/client.go b/pkg/cni/client.go index 6e7c9b794..bf863f5d5 100644 --- a/pkg/cni/client.go +++ b/pkg/cni/client.go @@ -23,6 +23,8 @@ import ( cnicurrent "github.com/containernetworking/cni/pkg/types/current" "github.com/davecgh/go-spew/spew" "github.com/golang/glog" + + "github.com/Mirantis/virtlet/pkg/utils" ) type Client struct { @@ -32,6 +34,7 @@ type Client struct { func NewClient(pluginsDir, configsDir string) (*Client, error) { configuration, err := ReadConfiguration(configsDir) + glog.V(3).Infof("CNI config: name: %q type: %q", configuration.Network.Name, configuration.Network.Type) if err != nil { return nil, fmt.Errorf("failed to read CNI configuration: %v", err) } @@ -42,18 +45,37 @@ func NewClient(pluginsDir, configsDir string) (*Client, error) { }, nil } +func (c *Client) Type() string { return c.configuration.Network.Type } + func (c *Client) cniRuntimeConf(podId, podName, podNs string) *libcni.RuntimeConf { - return &libcni.RuntimeConf{ + r := &libcni.RuntimeConf{ ContainerID: podId, NetNS: PodNetNSPath(podId), IfName: "virtlet-eth0", - Args: [][2]string{ + } + if podName != "" && podNs != "" { + r.Args = [][2]string{ {"IgnoreUnknown", "1"}, {"K8S_POD_NAMESPACE", podNs}, {"K8S_POD_NAME", podName}, {"K8S_POD_INFRA_CONTAINER_ID", podId}, - }, + } + } + return r +} + +// GetDummyNetwork creates a dummy network using CNI plugin. +// It's used for making a dummy gateway for Calico CNI plugin. +func (c *Client) GetDummyNetwork() (*cnicurrent.Result, error) { + // TODO: virtlet pod restarts should not grab another address for + // the gateway. That's not a big problem usually though + // as the IPs are not returned to Calico so both old + // IPs on existing VMs and new ones should work. + podId := utils.NewUuid() + if err := CreateNetNS(podId); err != nil { + return nil, fmt.Errorf("couldn't create netns for fake pod %q: %v", podId, err) } + return c.AddSandboxToNetwork(podId, "", "") } func (c *Client) AddSandboxToNetwork(podId, podName, podNs string) (*cnicurrent.Result, error) { diff --git a/pkg/tapmanager/tapfdsource.go b/pkg/tapmanager/tapfdsource.go index d3563b165..34d592c9c 100644 --- a/pkg/tapmanager/tapfdsource.go +++ b/pkg/tapmanager/tapfdsource.go @@ -18,7 +18,11 @@ package tapmanager import ( "encoding/json" + "errors" "fmt" + "net" + "os" + "strconv" "sync" "time" @@ -33,6 +37,12 @@ import ( "github.com/Mirantis/virtlet/pkg/nettools" ) +const ( + calicoNetType = "calico" + calicoDefaultSubnet = 24 + calicoSubnetVar = "VIRTLET_CALICO_SUBNET" +) + // PodNetworkDesc contains the data that are required by TapFDSource // to set up a tap device for a VM type PodNetworkDesc struct { @@ -70,8 +80,9 @@ type podNetwork struct { type TapFDSource struct { sync.Mutex - cniClient *cni.Client - fdMap map[string]*podNetwork + cniClient *cni.Client + dummyGateway net.IP + fdMap map[string]*podNetwork } var _ FDSource = &TapFDSource{} @@ -84,10 +95,28 @@ func NewTapFDSource(cniPluginsDir, cniConfigsDir string) (*TapFDSource, error) { return nil, err } - return &TapFDSource{ + s := &TapFDSource{ cniClient: cniClient, fdMap: make(map[string]*podNetwork), - }, nil + } + + // Calico needs special treatment here. + // We need to make network config DHCP-compatible by throwing away + // Calico's gateway and dev route and using a fake gateway instead. + // The fake gateway is just an IP address allocated by Calico IPAM, + // it's needed for proper ARP resppnses for VMs. + if cniClient.Type() == calicoNetType { + dummyResult, err := cniClient.GetDummyNetwork() + if err != nil { + return nil, err + } + if len(dummyResult.IPs) != 1 { + return nil, fmt.Errorf("expected 1 ip for the dummy network, but got %d", len(dummyResult.IPs)) + } + s.dummyGateway = dummyResult.IPs[0].Address.IP + } + + return s, nil } // GetFD implements GetFD method of FDSource interface @@ -121,6 +150,27 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { netConfig := payload.CNIConfig + // Calico needs network config to be adjusted for DHCP compatibility + if s.dummyGateway != nil { + if len(netConfig.IPs) != 1 { + return 0, nil, errors.New("didn't expect more than one IP config") + } + if netConfig.IPs[0].Version != "4" { + return 0, nil, errors.New("IPv4 config was expected") + } + netConfig.IPs[0].Address.Mask = netmaskForCalico() + netConfig.IPs[0].Gateway = s.dummyGateway + netConfig.Routes = []*cnitypes.Route{ + { + Dst: net.IPNet{ + IP: net.IP{0, 0, 0, 0}, + Mask: net.IPMask{0, 0, 0, 0}, + }, + GW: s.dummyGateway, + }, + } + } + netNSPath := cni.PodNetNSPath(pnd.PodId) vmNS, err := ns.GetNS(netNSPath) if err != nil { @@ -143,8 +193,10 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { // NOTE: older CNI plugins don't include the hardware address // in Result, but it's needed for Cloud-Init based - // network setup, so we add it here if it's missing - ensureCNIInterfaceHwAddress(netConfig, csn) + // network setup, so we add it here if it's missing. + // Also, some of the plugins may skip adding routes + // to the CNI result, so we must add them, too + fixCNIResult(netConfig, csn) // TODO: now CNIConfig should always contain interface mac address, so there // is no reason to pass it as separate field in dhcp.Config, @@ -244,7 +296,7 @@ func (s *TapFDSource) GetInfo(key string) ([]byte, error) { return pn.csn.HardwareAddr, nil } -func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) { +func fixCNIResult(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) { // If there's no interface info in netConfig, we can assume that we're dealing // with an old-style CNI plugin which only supports a single network interface if len(netConfig.Interfaces) > 0 { @@ -260,4 +312,21 @@ func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.Con for _, IP := range netConfig.IPs { IP.Interface = 0 } + + if len(netConfig.Routes) == 0 { + netConfig.Routes = csn.Result.Routes + } +} + +func netmaskForCalico() net.IPMask { + n := calicoDefaultSubnet + subnetStr := os.Getenv(calicoSubnetVar) + if subnetStr != "" { + n, err := strconv.Atoi(subnetStr) + if err != nil || n <= 0 || n > 30 { + glog.Warningf("bad calico subnet %q, using /%d", subnetStr, calicoDefaultSubnet) + n = calicoDefaultSubnet + } + } + return net.CIDRMask(n, 32) } diff --git a/tests/e2e/basic_test.go b/tests/e2e/basic_test.go index 9e5e14176..3330704a0 100644 --- a/tests/e2e/basic_test.go +++ b/tests/e2e/basic_test.go @@ -133,25 +133,43 @@ var _ = Describe("Basic cirros tests", func() { }) It("Should contain login string in VM log", func() { - out := do(framework.ExecSimple(nodeExecutor, "cat", - fmt.Sprintf("/var/log/virtlet/vms/%s/%s", sandboxID, filename))).(string) - Expect(strings.Count(out, - "login as 'cirros' user. default password: 'cubswin:)'. use 'sudo' for root.", - )).To(Equal(1)) + Eventually(func() error { + out, err := framework.ExecSimple(nodeExecutor, "cat", + fmt.Sprintf("/var/log/virtlet/vms/%s/%s", sandboxID, filename)) + if err != nil { + return err + } + fmt.Printf("OUT:\n%s\n---\n", out) + n := strings.Count(out, "login as 'cirros' user. default password: 'cubswin:)'. use 'sudo' for root.") + if n != 1 { + return fmt.Errorf("expected login prompt to appear exactly once in the log, but got %d occurences", n) + } + return nil + }, 60*5, 5) }) It("Should contain login string in pod log and each line of that log must be a valid JSON", func() { - out := do(framework.ExecSimple(nodeExecutor, "cat", - fmt.Sprintf("/var/log/pods/%s/%s", sandboxID, filename))).(string) - found := 0 - for _, line := range strings.Split(out, "\n") { - var entry map[string]string - Expect(json.Unmarshal([]byte(line), &entry)).To(Succeed()) - if strings.HasPrefix(entry["log"], "login as 'cirros' user. default password") { - found++ + Eventually(func() error { + out, err := framework.ExecSimple(nodeExecutor, "cat", + fmt.Sprintf("/var/log/pods/%s/%s", sandboxID, filename)) + if err != nil { + return err } - } - Expect(found).To(Equal(1)) + found := 0 + for _, line := range strings.Split(out, "\n") { + var entry map[string]string + if err := json.Unmarshal([]byte(line), &entry); err != nil { + return fmt.Errorf("error unmarshalling json: %v", err) + } + if strings.HasPrefix(entry["log"], "login as 'cirros' user. default password") { + found++ + } + } + if found != 1 { + return fmt.Errorf("expected login prompt to appear exactly once in the log, but got %d occurences", found) + } + return nil + }) }) })