Skip to content

Commit

Permalink
Kubernetes CNI plugin for EVE
Browse files Browse the repository at this point in the history
eve-bridge is a CNI plugin for Kubernetes which allows to connect
Pod with EVE Network Instances. This CNI plugin establishes
Pod connectivity in cooperation with the zedrouter microservice.
A CNI request received from Kubernetes to connect Pod to a given
network instance is delegated by eve-bridge to zedrouter via an RPC
call over an AF-UNIX socket. More details can be found in the
documentation submitted under pkg/kube/eve-bridge/README.md

Signed-off-by: Milan Lenco <milan@zededa.com>
  • Loading branch information
milan-zededa committed Mar 5, 2024
1 parent b5fd5b2 commit 7d2b5cb
Show file tree
Hide file tree
Showing 506 changed files with 223,627 additions and 322 deletions.
2 changes: 1 addition & 1 deletion images/modifiers/kubevirt.yq
Original file line number Diff line number Diff line change
@@ -1 +1 @@
.services |= .+ [{"name": "kube","image": "KUBE_TAG", "cgroupsPath": "/eve/services/kube", "oomScoreAdj": -999}]
.services += [{"name": "kube","image": "KUBE_TAG", "cgroupsPath": "/eve/services/kube", "oomScoreAdj": -999}]
11 changes: 9 additions & 2 deletions pkg/kube/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
# syntax=docker/dockerfile-upstream:1.5.0-rc2-labs

FROM lfedge/eve-alpine:57d6a8a768256bece2f03ae4fc0eedcb5bca41a6 as build
FROM lfedge/eve-alpine:a76c418ea841a1c433e0ab562653c3801f494d8e as build
ENV BUILD_PKGS go
ENV PKGS alpine-baselayout musl-utils iproute2 iptables curl openrc \
open-iscsi libvirt libvirt-client util-linux grep findutils jq
open-iscsi libvirt libvirt-client util-linux grep findutils jq \
cni-plugins
RUN eve-alpine-deploy.sh

COPY eve-bridge /plugins/eve-bridge
WORKDIR /plugins/eve-bridge
RUN GO111MODULE=on CGO_ENABLED=0 go build -v -ldflags "-s -w" -mod=vendor -o /out/usr/bin/eve-bridge .

FROM scratch
COPY --from=build /out/ /
COPY cluster-init.sh /usr/bin/
COPY cgconfig.conf /etc
# kubevirt yaml files are patched files and will be removed later, look at cluster-init.sh
COPY multus-daemonset.yaml /etc
COPY kubevirt-operator.yaml /etc
COPY kubevirt-features.yaml /etc
RUN mkdir -p /etc/containerd
Expand Down
120 changes: 90 additions & 30 deletions pkg/kube/cluster-init.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,59 +7,96 @@ K3S_VERSION=v1.26.3+k3s1
KUBEVIRT_VERSION=v0.59.0
LONGHORN_VERSION=v1.4.2
CDI_VERSION=v1.56.0
NODE_IP=""

INSTALL_LOG=/var/lib/install.log
CTRD_LOG=/var/lib/containerd.log
LOG_SIZE=$((5*1024*1024))

logmsg() {
local MSG
local TIME
MSG="$*"
TIME=$(date +"%F %T")
echo "$TIME : $MSG" >> $INSTALL_LOG
local MSG
local TIME
MSG="$*"
TIME=$(date +"%F %T")
echo "$TIME : $MSG" >> $INSTALL_LOG
}

setup_cgroup () {
echo "cgroup /sys/fs/cgroup cgroup defaults 0 0" >> /etc/fstab
echo "cgroup /sys/fs/cgroup cgroup defaults 0 0" >> /etc/fstab
}


check_network_connection () {
while true; do

ret=$(curl -o /dev/null -w "%{http_code}" -s "https://get.k3s.io")
if [ "$ret" -eq 200 ]; then
logmsg "Network is ready."
break;
else
logmsg "Network is not yet ready"
fi

sleep 5
done
while true; do
ret=$(curl -o /dev/null -w "%{http_code}" -s "https://get.k3s.io")
if [ "$ret" -eq 200 ]; then
logmsg "Network is ready."
break;
else
logmsg "Network is not yet ready"
fi
sleep 5
done
}

wait_for_default_route() {
while read -r iface dest gw flags refcnt use metric mask mtu window irtt; do
if [ "$dest" = "00000000" ] && [ "$mask" = "00000000" ]; then
logmsg "Default route found"
return 0
fi
#Make yetus happy
logmsg "waiting for default route $iface $dest $gw $flags $refcnt $use $metric $mask $mtu $window $irtt"
sleep 1
done < /proc/net/route

return 1
while read -r iface dest gw flags refcnt use metric mask mtu window irtt; do
if [ "$dest" = "00000000" ] && [ "$mask" = "00000000" ]; then
logmsg "Default route found"
return 0
fi
logmsg "waiting for default route $iface $dest $gw $flags $refcnt $use $metric $mask $mtu $window $irtt"
sleep 1
done < /proc/net/route
return 1
}

# Get IP of the interface with the first default route
# This will be then used as K3s node IP.
get_default_intf_IP_prefix() {
echo "Trying to obtain Node IP..."
while [ -z "$NODE_IP" ]; do
# Find the default route interface
default_interface="$(ip route show default | head -n 1 | awk '/default/ {print $5}')"
# Get the IP address of the default route interface
NODE_IP="$(ip addr show dev "$default_interface" | awk '/inet / {print $2}' | cut -d "/" -f1)"
[ -z "$NODE_IP" ] && sleep 1
done
echo "Node IP Address: $ip_address"

Check failure on line 65 in pkg/kube/cluster-init.sh

View workflow job for this annotation

GitHub Actions / yetus

shellcheck: warning: ip_address is referenced but not assigned. [SC2154]
ip_prefix="$NODE_IP/32"
# Fill in the outbound external Interface IP prefix in multus config
awk -v new_ip="$ip_prefix" '{gsub("IPAddressReplaceMe", new_ip)}1' /etc/multus-daemonset.yaml > /tmp/multus-daemonset.yaml
}

apply_multus_cni() {
get_default_intf_IP_prefix
kubectl create namespace eve-kube-app
logmsg "Apply Multus, Node-IP: $NODE_IP"
while ! kubectl apply -f /tmp/multus-daemonset.yaml; do
sleep 1
done
logmsg "Done applying Multus"
ln -s /var/lib/cni/bin/multus /var/lib/rancher/k3s/data/current/bin/multus
# need to only do this once
touch /var/lib/multus_initialized
}

copy_cni_plugin_files() {
mkdir -p /var/lib/cni/bin
mkdir -p /opt/cni/bin
cp /usr/libexec/cni/* /var/lib/cni/bin
cp /usr/libexec/cni/* /opt/cni/bin
cp /usr/bin/eve-bridge /var/lib/cni/bin
cp /usr/bin/eve-bridge /opt/cni/bin
logmsg "CNI plugins are installed"
}

wait_for_vault() {
logmsg "Starting wait for Vault"
pillarRootfs=/hostfs/containers/services/pillar/rootfs
while ! LD_LIBRARY_PATH=${pillarRootfs}/usr/lib/ ${pillarRootfs}/opt/zededa/bin/vaultmgr waitUnsealed;
do
sleep 1
sleep 1
done
logmsg "Vault ready"
}
Expand Down Expand Up @@ -178,6 +215,17 @@ if [ ! -f /var/lib/all_components_initialized ]; then
touch /var/lib/k3s_initialized
fi

if [ ! -f /var/lib/cni/bin ]; then
copy_cni_plugin_files
fi
if [ ! -f /var/lib/multus_initialized ]; then
apply_multus_cni
fi
if ! pidof dhcp; then
# launch CNI dhcp service
/opt/cni/bin/dhcp daemon &
fi

if [ ! -f /var/lib/kubevirt_initialized ]; then
# This patched version will be removed once the following PR https://github.com/kubevirt/kubevirt/pull/9668 is merged
logmsg "Installing patched Kubevirt"
Expand Down Expand Up @@ -224,6 +272,18 @@ else
logmsg "k3s is ready on this node"
# Default location where clients will look for config
ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config

# Initialize CNI after k3s reboot
if [ ! -f /var/lib/cni/bin ]; then
copy_cni_plugin_files
fi
if [ ! -f /var/lib/multus_initialized ]; then
apply_multus_cni
fi
if ! pidof dhcp; then
# launch CNI dhcp service
/opt/cni/bin/dhcp daemon &
fi
fi
fi
currentSize=$(wc -c <"$CTRD_LOG")
Expand Down
116 changes: 116 additions & 0 deletions pkg/kube/cnirpc/cnirpc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
// Copyright (c) 2023 Zededa, Inc.
// SPDX-License-Identifier: Apache-2.0

// Type definitions of arguments used for RPC methods called by eve-bridge CNI plugin.
// Imported by both eve-bridge (RPC client) and pillar/zedrouter (RPC server).

package cnirpc

import (
"net"

uuid "github.com/satori/go.uuid"
)

// CommonCNIRPCArgs : arguments used for every CNI RPC method
// (called by eve-bridge, served by zedrouter).
type CommonCNIRPCArgs struct {
Pod AppPod
// Interface inside the pod.
PodInterface NetInterfaceWithNs
}

// CommonCNIRPCRetval : a set of values returned by every CNI RPC method.
type CommonCNIRPCRetval struct {
AppUUID uuid.UUID
}

// ConnectPodAtL2Args : arguments for the ConnectPodAtL2 RPC method.
type ConnectPodAtL2Args struct {
CommonCNIRPCArgs
}

// ConnectPodAtL2Retval : type of the value returned by the ConnectPodAtL2 RPC method.
type ConnectPodAtL2Retval struct {
CommonCNIRPCRetval
UseDHCP bool
// Interfaces include the bridge interface and both sides of the VETH connecting
// pod with the host.
Interfaces []NetInterfaceWithNs
}

// ConnectPodAtL3Args : arguments for the ConnectPodAtL3 RPC method.
type ConnectPodAtL3Args struct {
CommonCNIRPCArgs
PodIPAMConfig
}

// ConnectPodAtL3Retval : type of the value returned by the ConnectPodAtL3 RPC method.
type ConnectPodAtL3Retval struct {
CommonCNIRPCRetval
}

// DisconnectPodArgs : arguments for the DisconnectPod RPC method.
type DisconnectPodArgs struct {
CommonCNIRPCArgs
}

// DisconnectPodRetval : type of the value returned by the DisconnectPod RPC method.
type DisconnectPodRetval struct {
CommonCNIRPCRetval
UsedDHCP bool
}

// CheckPodConnectionArgs : arguments for the CheckPodConnection RPC method.
type CheckPodConnectionArgs struct {
CommonCNIRPCArgs
}

// CheckPodConnectionRetval : type of the value returned by the CheckPodConnection RPC method.
type CheckPodConnectionRetval struct {
CommonCNIRPCRetval
UsesDHCP bool
}

// AppPod is defined only in the Kubernetes mode.
// It describes Kubernetes Pod under which a given app is running.
type AppPod struct {
Name string
// PodNetNsPath references network namespace of the Kubernetes pod
// inside which the application is running.
NetNsPath string
}

// NetInterfaceWithNs : single network interface (configured by zedrouter for Kube CNI).
type NetInterfaceWithNs struct {
Name string
MAC net.HardwareAddr
NetNsPath string
}

// PodIPAMConfig : IP config assigned to Pod by a Kubernetes IPAM plugin.
type PodIPAMConfig struct {
IPs []PodIPAddress
Routes []PodRoute
DNS PodDNS
}

// PodIPAddress : ip address assigned to kubernetes pod network interface.
type PodIPAddress struct {
Address *net.IPNet
Gateway net.IP
}

// PodRoute : network IP route configured for kubernetes pod network interface.
type PodRoute struct {
Dst *net.IPNet
GW net.IP
}

// PodDNS : settings for DNS resolver inside a kubernetes pod.
type PodDNS struct {
Nameservers []string
Domain string
Search []string
Options []string
}
5 changes: 5 additions & 0 deletions pkg/kube/cnirpc/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module github.com/lf-edge/eve/pkg/kube/cnirpc

go 1.20

require github.com/satori/go.uuid v1.2.0 // indirect
1 change: 1 addition & 0 deletions pkg/kube/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ etcd-expose-metrics: true
container-runtime-endpoint: "/run/containerd-user/containerd.sock"
etcd-arg:
- "quota-backend-bytes=8589934592"
disable-network-policy: true
Loading

0 comments on commit 7d2b5cb

Please sign in to comment.