-
Notifications
You must be signed in to change notification settings - Fork 4.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
tests: create basic test for creating VMs on github actions
We try to bring up 3 VMs and SSH to them.
- Loading branch information
Showing
8 changed files
with
499 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Copyright 2024 The Kubernetes Authors. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| set -o errexit | ||
| set -o nounset | ||
| set -o pipefail | ||
| set -o xtrace | ||
|
|
||
| REPO_ROOT=$(git rev-parse --show-toplevel) | ||
| cd ${REPO_ROOT}/tests/e2e/scenarios/bare-metal | ||
|
|
||
| cd ${REPO_ROOT} | ||
|
|
||
| systemctl disable --user qemu-vm0 || true | ||
| systemctl disable --user qemu-vm1 || true | ||
| systemctl disable --user qemu-vm2 || true | ||
|
|
||
| systemctl stop --user qemu-vm0 || true | ||
| systemctl stop --user qemu-vm1 || true | ||
| systemctl stop --user qemu-vm2 || true | ||
|
|
||
| sudo ip link del dev tap-vm0 || true | ||
| sudo ip link del dev tap-vm1 || true | ||
| sudo ip link del dev tap-vm2 || true | ||
|
|
||
| rm -rf .build/vm0 | ||
| rm -rf .build/vm1 | ||
| rm -rf .build/vm2 | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Copyright 2024 The Kubernetes Authors. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| set -o errexit | ||
| set -o nounset | ||
| set -o pipefail | ||
| set -o xtrace | ||
|
|
||
| REPO_ROOT=$(git rev-parse --show-toplevel) | ||
| cd ${REPO_ROOT} | ||
|
|
||
| ip route || true | ||
|
|
||
| ip link || true | ||
|
|
||
| ip addr || true | ||
|
|
||
|
|
||
| journalctl --user -u qemu-dhcp || true | ||
| journalctl --user -u qemu-vm1 || true | ||
| journalctl --user -u qemu-vm2 || true | ||
| journalctl --user -u qemu-vm3 || true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Copyright 2024 The Kubernetes Authors. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| set -o errexit | ||
| set -o nounset | ||
| set -o pipefail | ||
| set -o xtrace | ||
|
|
||
| REPO_ROOT=$(git rev-parse --show-toplevel) | ||
| cd ${REPO_ROOT}/tests/e2e/scenarios/bare-metal | ||
|
|
||
| function cleanup() { | ||
| echo "running dump-artifacts" | ||
| ./dump-artifacts || true | ||
|
|
||
| echo "running cleanup" | ||
| ./cleanup || true | ||
| } | ||
|
|
||
| trap cleanup EXIT | ||
|
|
||
| ./start-vms | ||
|
|
||
| echo "Waiting 30 seconds for VMs to start" | ||
| sleep 30 | ||
|
|
||
| # Remove from known-hosts in case of reuse | ||
| ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.10 || true | ||
| ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.11 || true | ||
| ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.12 || true | ||
|
|
||
| ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 uptime | ||
| ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.11 uptime | ||
| ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.12 uptime | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,194 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Copyright 2024 The Kubernetes Authors. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| set -o errexit | ||
| set -o nounset | ||
| set -o pipefail | ||
| set -o xtrace | ||
|
|
||
| REPO_ROOT=$(git rev-parse --show-toplevel) | ||
| cd ${REPO_ROOT}/tests/e2e/scenarios/bare-metal | ||
|
|
||
| WORKDIR=${REPO_ROOT}/.build | ||
|
|
||
| # Create SSH keys | ||
| mkdir -p ${WORKDIR}/.ssh | ||
| if [[ ! -f ${WORKDIR}/.ssh/id_ed25519 ]]; then | ||
| ssh-keygen -t ed25519 -f ${WORKDIR}/.ssh/id_ed25519 -N "" | ||
| fi | ||
|
|
||
| # Build software we need | ||
| cd ${REPO_ROOT}/tools/metal/dhcp | ||
| go build -o ${WORKDIR}/dhcp . | ||
| # Give permission to listen on ports < 1024 (some of like a partial suid binary) | ||
| sudo setcap cap_net_bind_service=ep ${WORKDIR}/dhcp | ||
|
|
||
| # Install software we need | ||
| if ! genisoimage --version; then | ||
| echo "Installing genisoimage" | ||
| sudo apt-get install --yes genisoimage | ||
| fi | ||
| if ! qemu-img --version; then | ||
| echo "Installing qemu-img (via qemu-utils)" | ||
| sudo apt-get install --yes qemu-utils | ||
| fi | ||
|
|
||
| # Download boot disk | ||
| cd ${WORKDIR} | ||
| if [[ ! -f debian-12-generic-amd64-20240702-1796.qcow2 ]]; then | ||
| echo "Downloading debian-12-generic-amd64-20240702-1796.qcow2" | ||
| wget --no-verbose -N https://cloud.debian.org/images/cloud/bookworm/20240702-1796/debian-12-generic-amd64-20240702-1796.qcow2 | ||
| fi | ||
|
|
||
| # Create bridge | ||
| bridge_name=br0 | ||
| if (! ip link show ${bridge_name}); then | ||
| sudo ip link add ${bridge_name} type bridge | ||
| sudo ip address add 10.123.45.0/24 dev ${bridge_name} | ||
|
|
||
| sudo ip link add name loop-${bridge_name} type dummy | ||
| sudo ip addr add 10.123.45.1/24 dev loop-${bridge_name} | ||
|
|
||
| sudo ip link set dev loop-${bridge_name} master ${bridge_name} | ||
| sudo ip link set dev loop-${bridge_name} up | ||
|
|
||
| # Enable packets from one VM on the bridge to another | ||
| sudo iptables -A FORWARD -i ${bridge_name} -o ${bridge_name} -j ACCEPT | ||
|
|
||
| # Enable packets from a VM to reach the real internet via NAT | ||
| sudo iptables -t nat -A POSTROUTING -s 10.123.45.0/24 ! -o ${bridge_name} -j MASQUERADE | ||
| sudo iptables -A FORWARD -o ${bridge_name} -m state --state RELATED,ESTABLISHED -j ACCEPT | ||
| sudo iptables -A FORWARD -i ${bridge_name} ! -o ${bridge_name} -j ACCEPT | ||
|
|
||
| sudo ip link set dev ${bridge_name} up | ||
| fi | ||
|
|
||
| # TODO: Create tuntap with user $(whoami)? | ||
|
|
||
|
|
||
| function start_dhcp() { | ||
| # Great guide here: https://wiki.gentoo.org/wiki/QEMU/Options | ||
| mkdir -p ~/.config/systemd/user | ||
| cat <<EOF > ~/.config/systemd/user/qemu-dhcp.service | ||
| [Unit] | ||
| Description=qemu-dhcp | ||
| After=network.target | ||
| [Service] | ||
| EnvironmentFile=/etc/environment | ||
| Type=exec | ||
| WorkingDirectory=${WORKDIR}/ | ||
| ExecStart=${WORKDIR}/dhcp | ||
| Restart=always | ||
| [Install] | ||
| WantedBy=default.target | ||
| EOF | ||
|
|
||
| systemctl --user daemon-reload | ||
| systemctl --user enable --now qemu-dhcp.service | ||
| } | ||
|
|
||
| function run_vm() { | ||
| vm_name=$1 | ||
| mac=$2 | ||
| #ssh_port=$2 | ||
|
|
||
| mkdir ${WORKDIR}/${vm_name}/ | ||
| cd ${WORKDIR}/${vm_name} | ||
| PUBKEY=$(cat ${WORKDIR}/.ssh/id_ed25519.pub) | ||
|
|
||
| cat <<EOF > user-data | ||
| #cloud-config | ||
| users: | ||
| - name: my_user | ||
| groups: adm, cdrom, sudo, dip, plugdev, lxd | ||
| shell: /bin/bash | ||
| sudo: ALL=(ALL) NOPASSWD:ALL | ||
| ssh_authorized_keys: | ||
| - ${PUBKEY} | ||
| - name: root | ||
| ssh_authorized_keys: | ||
| - ${PUBKEY} | ||
| hostname: ${vm_name} | ||
| manage_etc_hosts: true | ||
| locale: en_US.UTF-8 | ||
| timezone: Europe/Berlin | ||
| EOF | ||
|
|
||
| cat <<EOF > meta-data | ||
| instance-id: ${vm_name} | ||
| local-hostname: cloudimg | ||
| EOF | ||
|
|
||
| genisoimage -output seed.iso -volid cidata -joliet -rock user-data meta-data | ||
|
|
||
| qemu-img create -b ../debian-12-generic-amd64-20240702-1796.qcow2 -F qcow2 -f qcow2 root.qcow2 20G | ||
|
|
||
| sudo ip tuntap add dev tap-${vm_name} mode tap | ||
| sudo ip link set dev tap-${vm_name} master ${bridge_name} | ||
| sudo ip link set dev tap-${vm_name} up | ||
|
|
||
| # Great guide here: https://wiki.gentoo.org/wiki/QEMU/Options | ||
| mkdir -p ~/.config/systemd/user | ||
| cat <<EOF > ~/.config/systemd/user/qemu-${vm_name}.service | ||
| [Unit] | ||
| Description=qemu-${vm_name} | ||
| After=network.target | ||
| [Service] | ||
| EnvironmentFile=/etc/environment | ||
| Type=exec | ||
| WorkingDirectory=${WORKDIR}/${vm_name}/ | ||
| ExecStart=qemu-system-x86_64 \ | ||
| -machine type=q35,accel=kvm \ | ||
| -object rng-random,id=rng0,filename=/dev/urandom -device virtio-rng-pci,rng=rng0 \ | ||
| -m 1024 \ | ||
| -cpu host \ | ||
| -smp 4 \ | ||
| -nographic \ | ||
| -netdev tap,ifname=tap-${vm_name},id=net0,script=no,downscript=no \ | ||
| -device virtio-net-pci,netdev=net0,mac=${mac} \ | ||
| -device virtio-scsi-pci,id=scsi0 \ | ||
| -drive file=root.qcow2,id=hdd0,if=none \ | ||
| -device scsi-hd,drive=hdd0,bus=scsi0.0 \ | ||
| -drive id=cdrom0,if=none,format=raw,readonly=on,file=seed.iso \ | ||
| -device scsi-cd,bus=scsi0.0,drive=cdrom0 | ||
| Restart=always | ||
| [Install] | ||
| WantedBy=default.target | ||
| EOF | ||
|
|
||
| # TODO: | ||
| # -serial mon:stdio -append 'console=ttyS0' | ||
|
|
||
| systemctl --user daemon-reload | ||
| systemctl --user enable --now qemu-${vm_name}.service | ||
| } | ||
|
|
||
|
|
||
| start_dhcp | ||
|
|
||
| # Note: not all mac addresses are valid; 52:54:00 is the prefix reserved for qemu | ||
| run_vm vm0 52:54:00:44:55:0a | ||
| run_vm vm1 52:54:00:44:55:0b | ||
| run_vm vm2 52:54:00:44:55:0c | ||
|
|
||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| module github.com/kubernetes/kops/tools/metal/dhcp | ||
|
|
||
| go 1.22.6 | ||
|
|
||
| require github.com/insomniacslk/dhcp v0.0.0-20240812123929-b105c29bd1b5 | ||
|
|
||
| require ( | ||
| github.com/josharian/native v1.1.0 // indirect | ||
| github.com/pierrec/lz4/v4 v4.1.14 // indirect | ||
| github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 // indirect | ||
| golang.org/x/sys v0.18.0 // indirect | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= | ||
| github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
| github.com/insomniacslk/dhcp v0.0.0-20240812123929-b105c29bd1b5 h1:GkMacU5ftc+IEg1449N3UEy2XLDz58W4fkrRu2fibb8= | ||
| github.com/insomniacslk/dhcp v0.0.0-20240812123929-b105c29bd1b5/go.mod h1:KclMyHxX06VrVr0DJmeFSUb1ankt7xTfoOA35pCkoic= | ||
| github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= | ||
| github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= | ||
| github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= | ||
| github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY= | ||
| github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4= | ||
| github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= | ||
| github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= | ||
| github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= | ||
| github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= | ||
| github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | ||
| github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | ||
| github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= | ||
| github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= | ||
| github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gVBKXc2MVSZ4G/NnWLtzw4gNA= | ||
| github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264= | ||
| golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= | ||
| golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= | ||
| golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= | ||
| golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= | ||
| golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= | ||
| golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= | ||
| golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= | ||
| gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= | ||
| gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
Oops, something went wrong.