forked from openzfs/zfs
-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Split functional testings via github action matrix
This commit changes the workflow of the github actions. We split the workflow into different parts: 1) build zfs modules for Ubuntu 20.04 and 22.04 (~25m) 2) 2x zloop test (~10m) + 2x sanity test (~25m) 3) functional testings in parts 1..5 (each ~1h) - these could be triggered, when sanity tests are ok - currently I just start them all in the same time 4) cleanup and create summary When everything is fine, the full run with all testings should be done in around 2 hours. The codeql.yml and checkstyle.yml are not part in this circle. The testings are also modified a bit: - report info about CPU and checksum benchmarks - reset the debugging logs for each test - when some error occurred, we call dmesg with -c to get only the log output for the last failed test - we empty also the dbgsys Reviewed-by: George Melikov <mail@gmelikov.ru> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de> Closes openzfs#14078
- Loading branch information
Showing
13 changed files
with
477 additions
and
257 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
|
||
## The testings are done this way | ||
|
||
```mermaid | ||
flowchart TB | ||
subgraph CleanUp and Summary | ||
Part1-20.04-->CleanUp+nice+Summary | ||
Part2-20.04-->CleanUp+nice+Summary | ||
PartN-20.04-->CleanUp+nice+Summary | ||
Part1-22.04-->CleanUp+nice+Summary | ||
Part2-22.04-->CleanUp+nice+Summary | ||
PartN-22.04-->CleanUp+nice+Summary | ||
end | ||
subgraph Functional Testings | ||
functional-testing-20.04-->Part1-20.04 | ||
functional-testing-20.04-->Part2-20.04 | ||
functional-testing-20.04-->PartN-20.04 | ||
functional-testing-22.04-->Part1-22.04 | ||
functional-testing-22.04-->Part2-22.04 | ||
functional-testing-22.04-->PartN-22.04 | ||
end | ||
subgraph Sanity and zloop Testings | ||
sanity-checks-20.04-->functional-testing-20.04 | ||
sanity-checks-22.04-->functional-testing-22.04 | ||
zloop-checks-20.04-->functional | ||
zloop-checks-22.04-->functional | ||
end | ||
subgraph Code Checking + Building | ||
codeql.yml | ||
checkstyle.yml | ||
Build-Ubuntu-20.04-->sanity-checks-20.04 | ||
Build-Ubuntu-22.04-->sanity-checks-22.04 | ||
Build-Ubuntu-20.04-->zloop-checks-20.04 | ||
Build-Ubuntu-22.04-->zloop-checks-22.04 | ||
end | ||
``` | ||
|
||
|
||
1) build zfs modules for Ubuntu 20.04 and 22.04 (~15m) | ||
2) 2x zloop test (~10m) + 2x sanity test (~25m) | ||
3) functional testings in parts 1..5 (each ~1h) | ||
4) cleanup and create summary | ||
- content of summary depends on the results of the steps | ||
|
||
When everything runs fine, the full run should be done in | ||
about 2 hours. | ||
|
||
The codeql.yml and checkstyle.yml are not part in this circle. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,119 @@ | ||
#!/usr/bin/env bash | ||
|
||
# for runtime reasons we split functional testings into N parts | ||
# - use a define to check for missing tarfiles | ||
FUNCTIONAL_PARTS="4" | ||
|
||
ZTS_REPORT="tests/test-runner/bin/zts-report.py" | ||
chmod +x $ZTS_REPORT | ||
|
||
function output() { | ||
echo -e $* >> Summary.md | ||
} | ||
|
||
function error() { | ||
output ":bangbang: $* :bangbang:\n" | ||
} | ||
|
||
# this function generates the real summary | ||
# - expects a logfile "log" in current directory | ||
function generate() { | ||
# we issued some error already | ||
test ! -s log && return | ||
|
||
# for overview and zts-report | ||
cat log | grep '^Test' > list | ||
|
||
# error details | ||
awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; } | ||
/\[SKIP\]|\[PASS\]/{ show=0; } show' log > err | ||
|
||
# summary of errors | ||
if [ -s err ]; then | ||
output "<pre>" | ||
$ZTS_REPORT --no-maybes ./list >> Summary.md | ||
output "</pre>" | ||
|
||
# generate seperate error logfile | ||
ERRLOGS=$((ERRLOGS+1)) | ||
errfile="err-$ERRLOGS.md" | ||
echo -e "\n## $headline (debugging)\n" >> $errfile | ||
echo "<details><summary>Error Listing - with dmesg and dbgmsg</summary><pre>" >> $errfile | ||
dd if=err bs=999k count=1 >> $errfile | ||
echo "</pre></details>" >> $errfile | ||
else | ||
output "All tests passed :thumbsup:" | ||
fi | ||
|
||
output "<details><summary>Full Listing</summary><pre>" | ||
cat list >> Summary.md | ||
output "</pre></details>" | ||
|
||
# remove tmp files | ||
rm -f err list log | ||
} | ||
|
||
# check tarfiles and untar | ||
function check_tarfile() { | ||
if [ -f "$1" ]; then | ||
tar xf "$1" || error "Tarfile $1 returns some error" | ||
else | ||
error "Tarfile $1 not found" | ||
fi | ||
} | ||
|
||
# check logfile and concatenate test results | ||
function check_logfile() { | ||
if [ -f "$1" ]; then | ||
cat "$1" >> log | ||
else | ||
error "Logfile $1 not found" | ||
fi | ||
} | ||
|
||
# sanity | ||
function summarize_s() { | ||
headline="$1" | ||
output "\n## $headline\n" | ||
rm -rf testfiles | ||
check_tarfile "$2/sanity.tar" | ||
check_logfile "testfiles/log" | ||
generate | ||
} | ||
|
||
# functional | ||
function summarize_f() { | ||
headline="$1" | ||
output "\n## $headline\n" | ||
rm -rf testfiles | ||
for i in $(seq 1 $FUNCTIONAL_PARTS); do | ||
tarfile="$2/part$i.tar" | ||
check_tarfile "$tarfile" | ||
check_logfile "testfiles/log" | ||
done | ||
generate | ||
} | ||
|
||
# https://docs.github.com/en/enterprise-server@3.6/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits | ||
# Job summaries are isolated between steps and each step is restricted to a maximum size of 1MiB. | ||
# [ ] can not show all error findings here | ||
# [x] split files into smaller ones and create additional steps | ||
|
||
ERRLOGS=0 | ||
if [ ! -f Summary/Summary.md ]; then | ||
# first call, we do the default summary (~500k) | ||
echo -n > Summary.md | ||
summarize_s "Sanity Tests Ubuntu 20.04" Logs-20.04-sanity | ||
summarize_s "Sanity Tests Ubuntu 22.04" Logs-22.04-sanity | ||
summarize_f "Functional Tests Ubuntu 20.04" Logs-20.04-functional | ||
summarize_f "Functional Tests Ubuntu 22.04" Logs-22.04-functional | ||
|
||
cat Summary.md >> $GITHUB_STEP_SUMMARY | ||
mkdir -p Summary | ||
mv *.md Summary | ||
else | ||
# here we get, when errors where returned in first call | ||
test -f Summary/err-$1.md && cat Summary/err-$1.md >> $GITHUB_STEP_SUMMARY | ||
fi | ||
|
||
exit 0 |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -eu | ||
|
||
function prerun() { | ||
echo "::group::Install build dependencies" | ||
# remove snap things, update+upgrade will be faster then | ||
for x in lxd core20 snapd; do sudo snap remove $x; done | ||
sudo apt-get purge snapd google-chrome-stable firefox | ||
# https://github.com/orgs/community/discussions/47863 | ||
sudo apt-get remove grub-efi-amd64-bin grub-efi-amd64-signed shim-signed --allow-remove-essential | ||
sudo apt-get update | ||
sudo apt upgrade | ||
sudo xargs --arg-file=.github/workflows/build-dependencies.txt apt-get install -qq | ||
sudo apt-get clean | ||
sudo dmesg -c > /var/tmp/dmesg-prerun | ||
echo "::endgroup::" | ||
} | ||
|
||
function mod_build() { | ||
echo "::group::Generate debian packages" | ||
./autogen.sh | ||
./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan | ||
make --no-print-directory --silent native-deb-utils native-deb-kmod | ||
mv ../*.deb . | ||
rm ./openzfs-zfs-dracut*.deb ./openzfs-zfs-dkms*.deb | ||
echo "$ImageOS-$ImageVersion" > tests/ImageOS.txt | ||
echo "::endgroup::" | ||
} | ||
|
||
function mod_install() { | ||
# install the pre-built module only on the same runner image | ||
MOD=`cat tests/ImageOS.txt` | ||
if [ "$MOD" != "$ImageOS-$ImageVersion" ]; then | ||
rm -f *.deb | ||
mod_build | ||
fi | ||
|
||
echo "::group::Install and load modules" | ||
# delete kernel-shipped zfs modules, be sure about correct modules | ||
sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf | ||
sudo apt-get install --fix-missing ./*.deb | ||
|
||
# Native Debian packages enable and start the services | ||
# Stop zfs-zed daemon, as it may interfere with some ZTS test cases | ||
sudo systemctl stop zfs-zed | ||
sudo depmod -a | ||
sudo modprobe zfs | ||
sudo dmesg | ||
sudo dmesg -c > /var/tmp/dmesg-module-load | ||
echo "::endgroup::" | ||
|
||
echo "::group::Report CPU information" | ||
lscpu | ||
cat /proc/spl/kstat/zfs/chksum_bench | ||
echo "::endgroup::" | ||
|
||
echo "::group::Reclaim and report disk space" | ||
# remove 4GiB of images | ||
sudo systemd-run docker system prune --force --all --volumes | ||
|
||
# remove unused software | ||
sudo systemd-run --wait rm -rf \ | ||
"$AGENT_TOOLSDIRECTORY" \ | ||
/opt/* \ | ||
/usr/local/* \ | ||
/usr/share/az* \ | ||
/usr/share/dotnet \ | ||
/usr/share/gradle* \ | ||
/usr/share/miniconda \ | ||
/usr/share/swift \ | ||
/var/lib/gems \ | ||
/var/lib/mysql \ | ||
/var/lib/snapd | ||
|
||
# trim the cleaned space | ||
sudo fstrim / | ||
|
||
# disk usage afterwards | ||
df -h / | ||
echo "::endgroup::" | ||
} | ||
|
||
case "$1" in | ||
build) | ||
prerun | ||
mod_build | ||
;; | ||
tests) | ||
prerun | ||
mod_install | ||
;; | ||
esac |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -eu | ||
|
||
TDIR="/usr/share/zfs/zfs-tests/tests/functional" | ||
echo -n "TODO=" | ||
case "$1" in | ||
part1) | ||
# ~1h 20m | ||
echo "cli_root" | ||
;; | ||
part2) | ||
# ~1h | ||
ls $TDIR|grep '^[a-m]'|grep -v "cli_root"|xargs|tr -s ' ' ',' | ||
;; | ||
part3) | ||
# ~1h | ||
ls $TDIR|grep '^[n-qs-z]'|xargs|tr -s ' ' ',' | ||
;; | ||
part4) | ||
# ~1h | ||
ls $TDIR|grep '^r'|xargs|tr -s ' ' ',' | ||
;; | ||
esac |
Oops, something went wrong.