ZTS: Use QEMU for tests on Linux and FreeBSD

This commit adds functional tests for these systems:
- AlmaLinux 8, AlmaLinux 9
- ArchLinux
- CentOS Stream 9
- Fedora 39, Fedora 40
- Debian 11, Debian 12
- FreeBSD 13, FreeBSD 14, FreeBSD 15
- Ubuntu 20.04, Ubuntu 22.04, Ubuntu 24.04

Workflow for each operating system:
- install QEMU on the github runner
- download current cloud image
- start and init that image via cloud-init
- install deps and poweroff system
- start system and build openzfs and then poweroff again
- clone the system and start qemu workers for parallel testings
- do the functional testings, hopefully < 3h

Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>

.github/workflows/scripts/README.md

@@ -0,0 +1,9 @@
+
+Workflow for each operating system:
+- install qemu on the github runner
+- download current cloud image of operating system
+- start and init that image via cloud-init
+- install dependencies and poweroff system
+- start system and build openzfs and then poweroff again
+- clone build system and start 3 instances of it
+- the functional testings complete within times < 3h

.github/workflows/scripts/merge_summary.awk

@@ -0,0 +1,109 @@
+#!/bin/awk -f
+#
+# Merge multiple ZTS tests results summaries into a single summary.  This is
+# needed when you're running different parts of ZTS on different tests
+# runners or VMs.
+#
+# Usage:
+#
+#	./merge_summary.awk summary1.txt [summary2.txt] [summary3.txt] ...
+#
+#	or:
+#
+#	cat summary*.txt | ./merge_summary.awk
+#
+BEGIN {
+	i=-1
+	pass=0
+	fail=0
+	skip=0
+	state=""
+	cl=0
+	el=0
+	upl=0
+	ul=0
+
+	# Total seconds of tests runtime
+	total=0;
+}
+
+# Skip empty lines
+/^\s*$/{next}
+
+# Skip Configuration and Test lines
+/^Test:/{state=""; next}
+/Configuration/{state="";next}
+
+# When we see "test-runner.py" stop saving config lines, and
+# save test runner lines
+/test-runner.py/{state="testrunner"; runner=runner$0"\n"; next}
+
+# We need to differentiate the PASS counts from test result lines that start
+# with PASS, like:
+#
+#   PASS mv_files/setup
+#
+# Use state="pass_count" to differentiate
+#
+/Results Summary/{state="pass_count"; next}
+/PASS/{ if (state=="pass_count") {pass += $2}}
+/FAIL/{ if (state=="pass_count") {fail += $2}}
+/SKIP/{ if (state=="pass_count") {skip += $2}}
+/Running Time/{
+	state="";
+	running[i]=$3;
+	split($3, arr, ":")
+	total += arr[1] * 60 * 60;
+	total += arr[2] * 60;
+	total += arr[3]
+	next;
+}
+
+/Tests with results other than PASS that are expected/{state="expected_lines"; next}
+/Tests with result of PASS that are unexpected/{state="unexpected_pass_lines"; next}
+/Tests with results other than PASS that are unexpected/{state="unexpected_lines"; next}
+{
+	if (state == "expected_lines") {
+		expected_lines[el] = $0
+		el++
+	}
+
+	if (state == "unexpected_pass_lines") {
+		unexpected_pass_lines[upl] = $0
+		upl++
+	}
+	if (state == "unexpected_lines") {
+		unexpected_lines[ul] = $0
+		ul++
+	}
+}
+
+# Reproduce summary
+END {
+	print runner;
+	print "\nResults Summary"
+	print "PASS\t"pass
+	print "FAIL\t"fail
+	print "SKIP\t"skip
+	print ""
+	print "Running Time:\t"strftime("%T", total, 1)
+	if (pass+fail+skip > 0) {
+		percent_passed=(pass/(pass+fail+skip) * 100)
+	}
+	printf "Percent passed:\t%3.2f%", percent_passed
+
+	print "\n\nTests with results other than PASS that are expected:"
+	asort(expected_lines, sorted)
+	for (j in sorted)
+		print sorted[j]
+
+	print "\n\nTests with result of PASS that are unexpected:"
+	asort(unexpected_pass_lines, sorted)
+	for (j in sorted)
+		print sorted[j]
+
+	print "\n\nTests with results other than PASS that are unexpected:"
+	asort(unexpected_lines, sorted)
+	for (j in sorted)
+		print sorted[j]
+}

.github/workflows/scripts/qemu-1-setup.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 1) setup qemu instance on action runner
+######################################################################
+
+set -eu
+
+# install needed packages
+sudo apt-get update
+sudo apt-get install axel cloud-image-utils daemonize guestfs-tools \
+  ksmtuned virt-manager linux-modules-extra-`uname -r`
+
+# generate ssh keys
+rm -f ~/.ssh/id_ed25519
+ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
+
+# no need for some scheduler
+for i in /sys/block/s*/queue/scheduler; do
+  echo "none" | sudo tee $i > /dev/null
+done
+
+# this one is fast and mostly free
+sudo mount -o remount,rw,noatime,barrier=0 /mnt
+
+# we expect RAM shortage
+cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null
+KSM_MONITOR_INTERVAL=60
+
+# Millisecond sleep between ksm scans for 16Gb server.
+# Smaller servers sleep more, bigger sleep less.
+KSM_SLEEP_MSEC=10
+KSM_NPAGES_BOOST=300
+KSM_NPAGES_DECAY=-50
+KSM_NPAGES_MIN=64
+KSM_NPAGES_MAX=2048
+
+KSM_THRES_COEF=20
+KSM_THRES_CONST=2048
+
+LOGFILE=/var/log/ksmtuned.log
+DEBUG=1
+EOF
+
+sudo systemctl restart ksm
+sudo systemctl restart ksmtuned

.github/workflows/scripts/qemu-2-start.sh

@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 2) start qemu with some operating system, init via cloud-init
+######################################################################
+
+set -eu
+
+# short name used in zfs-qemu.yml
+OS="$1"
+
+# OS variant (virt-install --os-variant list)
+OSv=$OS
+
+# compressed with .zst extension
+FREEBSD="https://github.com/mcmilk/openzfs-freebsd-images/releases/download/v2024-07-24"
+URLzs=""
+
+# Ubuntu mirrors
+#UBMIRROR="https://cloud-images.ubuntu.com"
+#UBMIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images"
+UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
+
+case "$OS" in
+  almalinux8)
+    OSNAME="AlmaLinux 8"
+    URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  almalinux9)
+    OSNAME="AlmaLinux 9"
+    URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  archlinux)
+    OSNAME="Archlinux"
+    URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
+    ;;
+  centos-stream9)
+    OSNAME="CentOS Stream 9"
+    URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
+    ;;
+  debian11)
+    OSNAME="Debian 11"
+    URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2"
+    ;;
+  debian12)
+    OSNAME="Debian 12"
+    URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
+    ;;
+  fedora39)
+    OSNAME="Fedora 39"
+    OSv="fedora39"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2"
+    ;;
+  fedora40)
+    OSNAME="Fedora 40"
+    OSv="fedora39"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2"
+    ;;
+  freebsd13)
+    OSNAME="FreeBSD 13 (Stable)"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.3-STABLE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd13r)
+    OSNAME="FreeBSD 13 (Release)"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.3-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd14)
+    OSNAME="FreeBSD 14 (Stable)"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.1-STABLE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd14r)
+    OSNAME="FreeBSD 14 (Release)"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.1-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd15)
+    OSNAME="FreeBSD 15 (Current)"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-15.0-CURRENT.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  tumbleweed)
+    OSNAME="openSUSE Tumbleweed"
+    OSv="opensusetumbleweed"
+    MIRROR="http://opensuse-mirror-gce-us.susecloud.net"
+    URL="$MIRROR/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2"
+    ;;
+  ubuntu20)
+    OSNAME="Ubuntu 20.04"
+    OSv="ubuntu20.04"
+    URL="$UBMIRROR/focal/current/focal-server-cloudimg-amd64.img"
+    ;;
+  ubuntu22)
+    OSNAME="Ubuntu 22.04"
+    OSv="ubuntu22.04"
+    URL="$UBMIRROR/jammy/current/jammy-server-cloudimg-amd64.img"
+    ;;
+  ubuntu24)
+    OSNAME="Ubuntu 24.04"
+    OSv="ubuntu24.04"
+    URL="$UBMIRROR/noble/current/noble-server-cloudimg-amd64.img"
+    ;;
+  *)
+    echo "Wrong value for variable OS!"
+    exit 111
+    ;;
+esac
+
+# freebsd15 -> used in zfs-qemu.yml
+echo "$OS" > /var/tmp/os.txt
+
+# freebsd14.0 -> used for virt-install
+echo "$OSv" > /var/tmp/osvariant.txt
+
+# FreeBSD 15 (Current) -> used for summary
+echo "$OSNAME" > /var/tmp/osname.txt
+
+IMG="/mnt/tests/cloudimg.qcow2"
+DISK="/mnt/tests/openzfs.qcow2"
+sudo mkdir -p "/mnt/tests"
+sudo chown $(whoami) /mnt/tests
+
+if [ ! -z "$URLzs" ]; then
+  echo "Loading image $URLzs ..."
+  time axel -q -o "$IMG.zst" "$URLzs"
+  zstd -q -d --rm "$IMG.zst"
+else
+  echo "Loading image $URL ..."
+  time axel -q -o "$IMG" "$URL"
+fi
+
+qemu-img convert -q -f qcow2 -O qcow2 -c \
+  -o compression_type=zstd,cluster_size=256k $IMG $DISK
+rm -f $IMG
+
+echo "Resizing image to 60GiB ..."
+qemu-img resize -q $DISK 60G
+
+PUBKEY=`cat ~/.ssh/id_ed25519.pub`
+cat <<EOF > /tmp/user-data
+#cloud-config
+
+fqdn: $OS
+
+# user:zfs password:1
+users:
+- name: root
+  shell: $BASH
+- name: zfs
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  shell: $BASH
+  lock-passwd: false
+  passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/
+  ssh_authorized_keys:
+    - $PUBKEY
+
+growpart:
+  mode: auto
+  devices: ['/']
+  ignore_growroot_disabled: false
+EOF
+
+sudo virsh net-update default add ip-dhcp-host \
+  "<host mac='52:54:00:83:79:00' ip='192.168.122.10'/>" --live --config
+
+sudo virt-install \
+  --os-variant $OSv \
+  --name "openzfs" \
+  --cpu host-passthrough \
+  --virt-type=kvm --hvm \
+  --vcpus=4,sockets=1 \
+  --memory $((1024*12)) \
+  --memballoon model=virtio \
+  --graphics none \
+  --network bridge=virbr0,model=e1000,mac='52:54:00:83:79:00' \
+  --cloud-init user-data=/tmp/user-data \
+  --disk $DISK,bus=virtio,cache=none,format=qcow2,driver.discard=unmap \
+  --import --noautoconsole >/dev/null
+
+# in case the directory isn't there already
+mkdir -p $HOME/.ssh
+
+cat <<EOF >> $HOME/.ssh/config
+# no questions please
+StrictHostKeyChecking no
+
+# small timeout, used in while loops
+ConnectTimeout 1
+EOF