Split functional testings via github action matrix

This commit changes the workflow of the github actions.

We split the workflow into different parts:

1) build zfs modules for Ubuntu 20.04 and 22.04 (~25m)
2) 2x zloop test (~10m) + 2x sanity test (~25m)
3) functional testings in parts 1..5 (each ~1h)
4) cleanup and create summary

When everything is fine, the full run with all testings
should be done in around 2 hours.

The codeql.yml and checkstyle.yml are not part in this circle.

The testings are also modfied a bit:
- report info about CPU and checksum benchmarks
- reset the debugging logs for each test
  - when some error occurred, we call dmesg with -c to get
    only the log output for the last failed test
  - we empty also the dbgsys

Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>

.github/workflows/README.md

@@ -0,0 +1,51 @@
+
+## The testings are done this way
+
+```mermaid
+flowchart TB
+subgraph CleanUp and Summary
+  Part1-20.04-->CleanUp+nice+Summary
+  Part2-20.04-->CleanUp+nice+Summary
+  PartN-20.04-->CleanUp+nice+Summary
+  Part1-22.04-->CleanUp+nice+Summary
+  Part2-22.04-->CleanUp+nice+Summary
+  PartN-22.04-->CleanUp+nice+Summary
+end
+
+subgraph Functional Testings
+  functional-testing-20.04-->Part1-20.04
+  functional-testing-20.04-->Part2-20.04
+  functional-testing-20.04-->PartN-20.04
+  functional-testing-22.04-->Part1-22.04
+  functional-testing-22.04-->Part2-22.04
+  functional-testing-22.04-->PartN-22.04
+end
+
+subgraph Sanity and zloop Testings
+  sanity-checks-20.04-->functional-testing-20.04
+  sanity-checks-22.04-->functional-testing-22.04
+  zloop-checks-20.04-->functional
+  zloop-checks-22.04-->functional
+end
+
+subgraph Code Checking + Building
+  codeql.yml
+  checkstyle.yml
+  Build-Ubuntu-20.04-->sanity-checks-20.04
+  Build-Ubuntu-22.04-->sanity-checks-22.04
+  Build-Ubuntu-20.04-->zloop-checks-20.04
+  Build-Ubuntu-22.04-->zloop-checks-22.04
+end
+```
+
+
+1) build zfs modules for Ubuntu 20.04 and 22.04 (~15m)
+2) 2x zloop test (~10m) + 2x sanity test (~25m)
+3) functional testings in parts 1..5 (each ~1h)
+4) cleanup and create summary
+   - content of summary depends on the results of the steps
+
+When everything runs fine, the full run should be done in
+about 2 hours.
+
+The codeql.yml and checkstyle.yml are not part in this circle.

.github/workflows/build-dependencies.txt

@@ -2,56 +2,49 @@ acl
 alien
 attr
 autoconf
+automake
 bc
 build-essential
+bzip2
 curl
 dbench
 debhelper-compat
 dh-python
 dkms
 fakeroot
 fio
+gawk
 gdb
-gdebi
 git
 ksh
-lcov
-libacl1-dev
 libaio-dev
 libattr1-dev
 libblkid-dev
 libcurl4-openssl-dev
-libdevmapper-dev
 libelf-dev
 libffi-dev
-libmount-dev
 libpam0g-dev
-libselinux1-dev
 libssl-dev
 libtool
 libudev-dev
-linux-headers-generic
 lsscsi
 mdadm
 nfs-kernel-server
-pamtester
 parted
-po-debconf
+pax
+pv
 python3
 python3-all-dev
 python3-cffi
 python3-dev
 python3-packaging
-python3-pip
 python3-setuptools
 python3-sphinx
-rng-tools-debian
-rsync
+quota
+rng-tools
 samba
+selinux-utils
 sysstat
+util-linux
 uuid-dev
-watchdog
-wget
-xfslibs-dev
-xz-utils
 zlib1g-dev

.github/workflows/scripts/generate-summary.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -eu
+
+ZFS_COLOR="scripts/zfs-tests-color.sh"
+ZTS_REPORT="tests/test-runner/bin/zts-report.py"
+chmod +x $ZTS_REPORT
+
+# all output goes to summary page
+function generate() {
+  LOG="$1"
+  test -f $LOG || return
+
+  # for overview and zts-resport
+  cat $LOG | grep '^Test: ' > list
+
+  # error details
+  awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }
+    /\[SKIP\]|\[PASS\]/{ show=0; } show' $LOG > err
+
+  # first the summary
+  echo -e "\n## $TEST\n"
+  if [ -s err ]; then
+    echo "<pre>"
+    $ZTS_REPORT --no-maybes ./list
+    echo "</pre>"
+    echo "<details><summary>Error Listings</summary><pre>"
+    cat err
+    echo "</pre></details>"
+  else
+    echo "No errors within these testings :thumbsup:"
+  fi
+
+  # xxx: use stores step-id+output here (colors)
+  echo "<details><summary>All Tests</summary><pre>"
+  cat list
+  echo "</pre></details>"
+  rm -f err list
+}
+
+function summarize() {
+  LOGDIR=$1
+  TEST="$2"
+  echo "::group::Generating summary $LOGDIR"
+  for p in $LOGDIR/sanity.tar; do test -f $p && tar xf $p; done
+  generate sanity/log >> Summary.md
+  cat Summary.md >> $GITHUB_STEP_SUMMARY
+  rm -rf sanity
+  echo "::endgroup::"
+}
+
+function summarize_parts() {
+  LOGDIR=$1
+  TEST="$2"
+  echo "::group::Generating summary $LOGDIR"
+  # generate all in one files from the part ones
+  for p in $LOGDIR/part[123456789].tar; do
+    test -f $p && tar xf $p
+  done
+  for p in part[123456789]/log; do
+    test -f $p && cat $p >> log
+  done
+  generate log >> Summary.md
+  cat Summary.md >> $GITHUB_STEP_SUMMARY
+  rm -rf part* log
+  echo "::endgroup::"
+}
+
+echo > Summary.md
+summarize Logs-20.04-sanity "Sanity Tests Ubuntu 20.04"
+summarize Logs-22.04-sanity "Sanity Tests Ubuntu 22.04"
+summarize_parts Logs-20.04-functional "Functional Tests Ubuntu 20.04"
+summarize_parts Logs-22.04-functional "Functional Tests Ubuntu 22.04"

.github/workflows/scripts/setup-dependencies.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+set -eu
+
+function prerun() {
+  echo "::group::Install build dependencies"
+  # remove snap things, update+upgrade will be faster then
+  for x in lxd core20 snapd; do sudo snap remove $x; done
+  sudo apt-get purge snapd google-chrome-stable firefox
+  # https://github.com/orgs/community/discussions/47863
+  sudo apt-get remove grub-efi-amd64-bin grub-efi-amd64-signed shim-signed --allow-remove-essential
+  sudo apt-get update
+  sudo apt upgrade
+  sudo xargs --arg-file=.github/workflows/build-dependencies.txt apt-get install -qq
+  sudo apt-get clean
+  sudo dmesg -c > /var/tmp/dmesg-prerun
+  echo "::endgroup::"
+}
+
+function mod_build() {
+  echo "::group::Generate debian packages"
+  ./autogen.sh
+  ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
+  make --no-print-directory --silent native-deb-utils native-deb-kmod
+  mv ../*.deb .
+  rm ./openzfs-zfs-dracut*.deb ./openzfs-zfs-dkms*.deb
+  echo "$ImageOS-$ImageVersion" > tests/ImageOS.txt
+  echo "::endgroup::"
+}
+
+function mod_install() {
+  # install the pre-built module only on the same runner image
+  MOD=`cat tests/ImageOS.txt`
+  if [ "$MOD" != "$ImageOS-$ImageVersion" ]; then
+    rm -f *.deb
+    mod_build
+  fi
+
+  echo "::group::Install and load modules"
+  # delete kernel-shipped zfs modules, be sure about correct modules
+  sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
+  sudo apt-get install --fix-missing ./*.deb
+  # our modules are in kernel/extra/zcommon/*.ko
+  sudo rm -rf /lib/modules/*/kernel/zfs/*.ko
+  sudo depmod -a
+  sudo modprobe zfs
+  sudo dmesg
+  sudo dmesg -c > /var/tmp/dmesg-module-load
+  echo "::endgroup::"
+
+  echo "::group::Report CPU information"
+  lscpu
+  cat /proc/spl/kstat/zfs/chksum_bench
+  echo "::endgroup::"
+
+  echo "::group::Reclaim and report disk space"
+  # remove 4GiB of images
+  sudo systemd-run docker system prune --force --all --volumes
+
+  # remove unused software
+  sudo systemd-run --wait rm -rf \
+    "$AGENT_TOOLSDIRECTORY" \
+    /opt/* \
+    /usr/local/* \
+    /usr/share/az* \
+    /usr/share/dotnet \
+    /usr/share/gradle* \
+    /usr/share/miniconda \
+    /usr/share/swift \
+    /var/lib/gems \
+    /var/lib/mysql \
+    /var/lib/snapd
+
+  # trim the cleaned space
+  sudo fstrim /
+
+  # disk usage afterwards
+  df -h /
+  echo "::endgroup::"
+}
+
+case "$1" in
+  build)
+    prerun
+    mod_build
+    ;;
+  tests)
+    prerun
+    mod_install
+    ;;
+esac

.github/workflows/scripts/setup-functional.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -eu
+
+TDIR="/usr/share/zfs/zfs-tests/tests/functional"
+echo -n "TODO="
+case "$1" in
+  part1)
+    echo "cli_root"
+  ;;
+  part2)
+    ls $TDIR|grep '^[a-k]'|grep -v "cli_root"|xargs|tr -s ' ' ','
+  ;;
+  part3)
+    ls $TDIR|grep '^[l-q]'|xargs|tr -s ' ' ','
+  ;;
+  part4)
+    ls $TDIR|grep '^r[aeo]'|xargs|tr -s ' ' ','
+  ;;
+  part5)
+    echo -n "rsend,"
+    ls $TDIR|grep '^[s-z]'|xargs|tr -s ' ' ','
+  ;;
+esac