ZTS: Use QEMU for tests on Linux and FreeBSD

This commit adds functional tests for these systems:
- AlmaLinux 8, AlmaLinux 9
- ArchLinux
- CentOS Stream 9
- Fedora 39, Fedora 40
- Debian 11, Debian 12
- FreeBSD 13, FreeBSD 14, FreeBSD 15
- Ubuntu 22.04, Ubuntu 24.04

- enabled by default:
  - AlmaLinux 8, AlmaLinux 9
  - Fedora 39, Fedora 40
  - Debian 11
  - FreeBSD 13, FreeBSD 14, FreeBSD 15

Workflow for each operating system:
- install QEMU on the github runner
- download current cloud image
- start and init that image via cloud-init
- install deps and poweroff system
- start system and build openzfs and then poweroff again
- clone build system and start 3 qemu machines for tests
- using trimable virtual disks would also be possible
  - warning: tests seembuggy on virtio based disks (scsi and block disk)
- the functional testings complete in times < 3h

Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>
Co-authored-by: Tony Hutter <hutter2@llnl.gov>

.github/workflows/scripts/README.md

@@ -0,0 +1,10 @@
+
+Workflow for each operating system:
+ - install QEMU on the github runner
+ - download current cloud image
+ - start and init that image via cloud-init
+ - install deps and poweroff system
+ - start system and build openzfs and then poweroff again
+ - clone the system and start 4 qemu workers for the testings (4x 3GB RAM)
+ - use trimable virtual disks (3x 1GB) for each testing system
+ - do the functional testings < 3h for each os

.github/workflows/scripts/merge_summary.awk

@@ -0,0 +1,104 @@
+#!/bin/awk -f
+#
+# Merge multiple ZTS tests results summaries into a single summary.  This is
+# needed when you're running different parts of ZTS on different tests
+# runners or VMs.
+#
+# Usage:
+#
+#	./merge_summary.awk summary1.txt [summary2.txt] [summary3.txt] ...
+#
+#	or:
+#
+#	cat summary*.txt | ./merge_summary.awk
+#
+BEGIN {
+	i=-1
+	pass=0
+	fail=0
+	skip=0
+	state=""
+	cl=0
+	el=0
+	upl=0
+	ul=0
+
+	# Total seconds of tests runtime
+	total=0;
+}
+
+# Skip empty lines
+/^\s*$/{next}
+
+# Skip Configuration and Test lines
+/^Test:/{state=""; next}
+/Configuration/{state="";next}
+
+# When we see "test-runner.py" stop saving config lines, and
+# save test runner lines
+/test-runner.py/{state="testrunner"; runner=runner$0"\n"; next}
+
+# We need to differentiate the PASS counts from test result lines that start
+# with PASS, like:
+#
+#   PASS mv_files/setup
+#
+# Use state="pass_count" to differentiate
+#
+/Results Summary/{state="pass_count"; next}
+/PASS/{ if (state=="pass_count") {pass += $2}}
+/FAIL/{ if (state=="pass_count") {fail += $2}}
+/SKIP/{ if (state=="pass_count") {skip += $2}}
+/Running Time/{
+	state="";
+	running[i]=$3;
+	split($3, arr, ":")
+	total += arr[1] * 60 * 60;
+	total += arr[2] * 60;
+	total += arr[3]
+	next;
+}
+
+/Tests with results other than PASS that are expected/{state="expected_lines"; next}
+/Tests with result of PASS that are unexpected/{state="unexpected_pass_lines"; next}
+/Tests with results other than PASS that are unexpected/{state="unexpected_lines"; next}
+{
+	if (state == "expected_lines") {
+		expected_lines[el] = $0
+		el++
+	}
+
+	if (state == "unexpected_pass_lines") {
+		unexpected_pass_lines[upl] = $0
+		upl++
+	}
+	if (state == "unexpected_lines") {
+		unexpected_lines[ul] = $0
+		ul++
+	}
+}
+
+# Reproduce summary
+END {
+	print runner;
+	print "\nResults Summary"
+	print "PASS\t"pass
+	print "FAIL\t"fail
+	print "SKIP\t"skip
+	print ""
+	print "Running Time:\t"strftime("%T", total, 1)
+	if (pass+fail+skip > 0) {
+		percent_passed=(pass/(pass+fail+skip) * 100)
+	}
+	printf "Percent passed:\t%3.2f%", percent_passed
+
+	print "\n\nTests with results other than PASS that are expected:"
+	for (j in expected_lines)
+		print expected_lines[j]
+	print "\n\nTests with result of PASS that are unexpected:"
+	for (j in unexpected_pass_lines)
+		print unexpected_pass_lines[j]
+	print "\n\nTests with results other than PASS that are unexpected:"
+	for (j in unexpected_lines)
+		print unexpected_lines[j]
+}

.github/workflows/scripts/qemu-1-setup.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 1) setup the action runner to start some qemu instance
+######################################################################
+
+set -eu
+
+# docker isn't needed, free some memory
+sudo systemd-run --wait docker system prune --force --all --volumes
+sudo systemctl stop docker.socket
+sudo apt-get remove docker-ce-cli docker-ce podman
+
+# remove unneeded things
+sudo apt-get remove google-chrome-stable snapd
+
+# install needed packages
+sudo apt-get update
+sudo apt-get install axel cloud-image-utils daemonize guestfs-tools \
+  virt-manager linux-modules-extra-`uname -r`
+
+# remove unused software
+df -h /
+sudo systemd-run --wait rm -rf \
+  /opt/* \
+  /usr/local/* \
+  /usr/share/az* \
+  /usr/share/dotnet \
+  /usr/share/gradle* \
+  /usr/share/miniconda \
+  /usr/share/swift \
+  /var/lib/gems \
+  /var/lib/mysql \
+  /var/lib/snapd
+
+# disk usage afterwards
+sudo df -h /
+sudo df -h /mnt
+sudo fstrim -a
+
+# generate ssh keys
+rm -f ~/.ssh/id_ed25519
+ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""

.github/workflows/scripts/qemu-2-start.sh

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 2) start qemu with some operating system, init via cloud-init
+######################################################################
+
+set -eu
+
+# valid ostypes: virt-install --os-variant list
+OS="$1"
+OSv=$OS
+
+# compressed with .zst extension
+FREEBSD="https://github.com/mcmilk/openzfs-freebsd-images/releases/download/2024-06-23"
+URLzs=""
+
+case "$OS" in
+  almalinux8)
+    OSNAME="AlmaLinux 8"
+    URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  almalinux9)
+    OSNAME="AlmaLinux 9"
+    URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  archlinux)
+    OSNAME="Archlinux"
+    URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
+    ;;
+  centos-stream9)
+    OSNAME="CentOS Stream 9"
+    URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
+    ;;
+  debian11)
+    OSNAME="Debian 11"
+    URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2"
+    ;;
+  debian12)
+    OSNAME="Debian 12"
+    URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
+    ;;
+  fedora39)
+    OSNAME="Fedora 39"
+    OSv="fedora39"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2"
+    ;;
+  fedora40)
+    OSNAME="Fedora 40"
+    OSv="fedora39"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2"
+    ;;
+  freebsd13)
+    OSNAME="FreeBSD 13"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.3-STABLE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd14)
+    OSNAME="FreeBSD 14"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.1-STABLE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd15)
+    OSNAME="FreeBSD 15"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-15.0-CURRENT.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  tumbleweed)
+    OSNAME="openSUSE Tumbleweed"
+    OSv="opensusetumbleweed"
+    MIRROR="http://opensuse-mirror-gce-us.susecloud.net"
+    URL="$MIRROR/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2"
+    ;;
+  ubuntu22)
+    OSNAME="Ubuntu 22.04"
+    OSv="ubuntu22.04"
+    MIRROR="https://cloud-images.ubuntu.com"
+    MIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images"
+    MIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
+    URL="$MIRROR/jammy/current/jammy-server-cloudimg-amd64.img"
+    ;;
+  ubuntu24)
+    OSNAME="Ubuntu 24.04"
+    OSv="ubuntu24.04"
+    #URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+    #URL="https://mirrors.cloud.tencent.com/ubuntu-cloud-images/noble/current/noble-server-cloudimg-amd64.img"
+    URL="https://mirror.citrahost.com/ubuntu-cloud-images/noble/current/noble-server-cloudimg-amd64.img"
+    ;;
+  *)
+    echo "Wrong value for variable OS!"
+    exit 111
+    ;;
+esac
+
+IMG="/mnt/cloudimg.qcow2"
+DISK="/mnt/openzfs.qcow2"
+sudo chown -R $(whoami) /mnt
+
+if [ ! -z "$URLzs" ]; then
+  echo "Loading image $URLzs ..."
+  time axel -q -o "$IMG.zst" "$URLzs" || exit 111
+  zstd -q -d --rm "$IMG.zst"
+else
+  echo "Loading image $URL ..."
+  time axel -q -o "$IMG" "$URL" || exit 111
+fi
+
+# for later use
+echo "$OS" > /var/tmp/os.txt
+echo "$OSv" > /var/tmp/osvariant.txt
+echo "$OSNAME" > /var/tmp/osname.txt
+
+# we use zstd for faster IO on the testing runner
+echo "Converting image ..."
+qemu-img convert -q -f qcow2 -O qcow2 -c \
+  -o compression_type=zstd,preallocation=off $IMG $DISK || exit 111
+rm -f $IMG || exit 111
+
+echo "Resizing image to 50GiB ..."
+qemu-img resize -q $DISK 50G || exit 111
+
+PUBKEY=`cat ~/.ssh/id_ed25519.pub`
+cat <<EOF > /tmp/user-data
+#cloud-config
+
+fqdn: $OS
+
+# user:zfs password:1
+users:
+- name: root
+  shell: $BASH
+- name: zfs
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  shell: $BASH
+  lock-passwd: false
+  passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/
+  ssh_authorized_keys:
+    - $PUBKEY
+
+growpart:
+  mode: auto
+  devices: ['/']
+  ignore_growroot_disabled: false
+EOF
+
+for i in `seq 0 3`; do
+  sudo virsh net-update default add ip-dhcp-host \
+    "<host mac='52:54:00:83:79:0$i' ip='192.168.122.1$i'/>" --live --config
+done
+
+sudo virt-install \
+  --os-variant $OSv \
+  --name "openzfs" \
+  --cpu host-passthrough \
+  --virt-type=kvm --hvm \
+  --vcpus=4,sockets=1 \
+  --memory $((1024*4)) \
+  --memballoon model=none \
+  --graphics none \
+  --network bridge=virbr0,model=e1000,mac='52:54:00:83:79:00' \
+  --cloud-init user-data=/tmp/user-data \
+  --controller=scsi,model=virtio-scsi \
+  --disk "$DISK",bus=scsi,format=qcow2,driver.discard=unmap \
+  --import --noautoconsole