Move webstandard script to .github directory

Also added:
- More MySQL debugging.
- Prevent failing on HTTP404 errors, as we should only fail on HTTP500.
- Test for both with and without example data.
  This is to make sure that when we have no entities for something (which
  we rarely test as we always upload data) the pages still look decent.
  As this script was used before we keep the old behaviour of always
  installing the data when there is no explicit database method given.

.github/jobs/baseinstall.sh

@@ -0,0 +1,122 @@
+#!/bin/sh
+
+. .github/jobs/ci_settings.sh
+
+export version="$1"
+db=${2:-install}
+
+set -eux
+
+PHPVERSION=$(php -r 'echo PHP_MAJOR_VERSION.".".PHP_MINOR_VERSION."\n";')
+export PHPVERSION
+
+section_start "Run composer"
+export APP_ENV="dev"
+composer install --no-scripts |tee "$ARTIFACTS"/composer_out.txt
+section_end
+
+section_start "Set simple admin password"
+echo "password" > ./etc/initial_admin_password.secret
+echo "default login admin password password" > ~/.netrc
+section_end
+
+section_start "Install domserver"
+make configure
+./configure \
+  --with-baseurl='https://localhost/domjudge/' \
+  --with-domjudge-user=root \
+  --enable-doc-build=no \
+  --enable-judgehost-build=no | tee "$ARTIFACTS"/configure.txt
+
+make domserver
+make install-domserver
+section_end
+
+section_start "SQL settings"
+cat > ~/.my.cnf <<EOF
+[client]
+host=sqlserver
+user=root
+password=root
+EOF
+cat ~/.my.cnf
+
+mysql_root "CREATE DATABASE IF NOT EXISTS \`domjudge\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+mysql_root "CREATE USER IF NOT EXISTS \`domjudge\`@'%' IDENTIFIED BY 'domjudge';"
+mysql_root "GRANT SELECT, INSERT, UPDATE, DELETE ON \`domjudge\`.* TO 'domjudge'@'%';"
+mysql_root "FLUSH PRIVILEGES;"
+
+# Show some MySQL debugging
+mysql_root "show databases"
+mysql_root "SELECT CURRENT_USER();"
+mysql_root "SELECT USER();"
+mysql_root "SELECT user,host FROM mysql.user"
+echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret
+mysql_user "SELECT CURRENT_USER();"
+mysql_user "SELECT USER();"
+section_end
+
+section_start "Install DOMjudge database"
+/opt/domjudge/domserver/bin/dj_setup_database -uroot -proot bare-install
+#>> "$ARTIFACTS"/mysql.txt
+# We don't have -o pipefail so the `tee` eats away the potential error.
+#cat "$ARTIFACTS"/mysql.txt
+section_end
+
+section_start "Show PHP config"
+php -v | tee -a "$ARTIFACTS"/php.txt
+php -m | tee -a "$ARTIFACTS"/php.txt
+section_end
+
+section_start "Show general config"
+printenv | tee -a "$ARTIFACTS"/environment.txt
+cp /etc/os-release "$ARTIFACTS"/os-release.txt
+cp /proc/cmdline "$ARTIFACTS"/cmdline.txt
+section_end
+
+section_start "Setup webserver"
+cp /opt/domjudge/domserver/etc/domjudge-fpm.conf /etc/php/"$PHPVERSION"/fpm/pool.d/domjudge.conf
+
+rm -f /etc/nginx/sites-enabled/*
+cp /opt/domjudge/domserver/etc/nginx-conf /etc/nginx/sites-enabled/domjudge
+
+openssl req -nodes -new -x509 -keyout /tmp/server.key -out /tmp/server.crt -subj "/C=NL/ST=Noord-Holland/L=Amsterdam/O=TestingForPR/CN=localhost"
+cp /tmp/server.crt /usr/local/share/ca-certificates/
+update-ca-certificates
+# shellcheck disable=SC2002
+cat "$(pwd)/.github/jobs/data/nginx_extra" | tee -a /etc/nginx/sites-enabled/domjudge
+nginx -t
+section_end
+
+section_start "Show webserver is up"
+for service in nginx php${PHPVERSION}-fpm; do
+    service "$service" restart
+    service "$service" status
+done
+section_end
+
+if [ "${db}" = "install" ]; then
+    section_start "Install the example data"
+    /opt/domjudge/domserver/bin/dj_setup_database -uroot -proot install-examples | tee -a "$ARTIFACTS/mysql.txt"
+    section_end
+fi
+
+section_start "Setup user"
+# We're using the admin user in all possible roles
+mysql_root "DELETE FROM userrole WHERE userid=1;" domjudge
+if [ "$version" = "team" ]; then
+    # Add team to admin user
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" domjudge
+    mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" domjudge
+elif [ "$version" = "jury" ]; then
+    # Add jury to admin user
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 2);" domjudge
+elif [ "$version" = "balloon" ]; then
+    # Add balloon to admin user
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 4);" domjudge
+elif [ "$version" = "admin" ]; then
+    # Add admin to admin user
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" domjudge
+fi
+section_end
+

.github/jobs/ci_settings.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Store artifacts/logs
+export ARTIFACTS="/tmp/artifacts"
+mkdir -p "$ARTIFACTS"
+
+# Functions to annotate the Github actions logs
+trace_on () {
+    set -x
+}
+trace_off () {
+    {
+        set +x
+    } 2>/dev/null
+}
+
+section_start_internal () {
+    echo "::group::$1"
+    trace_on
+}
+
+section_end_internal () {
+    echo "::endgroup::"
+    trace_on
+}
+
+mysql_root () {
+    # shellcheck disable=SC2086
+    echo "$1" | mysql -uroot -proot ${2:-} | tee -a "$ARTIFACTS"/mysql.txt
+}
+
+mysql_user () {
+    # shellcheck disable=SC2086
+    echo "$1" | mysql -udomjudge -pdomjudge ${2:-} | tee -a "$ARTIFACTS"/mysql.txt
+}
+
+section_start () {
+    if [ "$#" -ne 1 ]; then
+        echo "Only 1 argument is needed for GHA, 2 was needed for GitLab."
+        exit 1
+    fi
+    trace_off
+    section_start_internal "$1"
+}
+
+section_end () {
+    trace_off
+    section_end_internal
+}

.github/jobs/pa11y_config.json

@@ -0,0 +1,9 @@
+{
+  "chromeLaunchConfig": {
+    "args": [
+      "--no-sandbox",
+      "--disable-setuid-sandbox",
+      "--disable-dev-shm-usage"
+    ]
+  }
+}

.github/jobs/webstandard.sh

@@ -0,0 +1,153 @@
+#!/bin/bash
+
+. .github/jobs/ci_settings.sh
+
+DIR="$PWD"
+
+if [ "$#" -ne "2" ]; then
+    exit 2
+fi
+
+TEST="$1"
+ROLE="$2"
+
+cd /opt/domjudge/domserver
+
+section_start "Setup pa11y"
+pa11y --version
+section_end
+
+section_start "Setup the test user"
+ADMINPASS=$(cat etc/initial_admin_password.secret)
+export COOKIEJAR
+COOKIEJAR=$(mktemp --tmpdir)
+export CURLOPTS="--fail -sq -m 30 -b $COOKIEJAR"
+if [ "$ROLE" = "public" ]; then
+    ADMINPASS="failedlogin"
+fi
+
+# Make an initial request which will get us a session id, and grab the csrf token from it
+CSRFTOKEN=$(curl $CURLOPTS -c $COOKIEJAR "http://localhost/domjudge/login" 2>/dev/null | sed -n 's/.*_csrf_token.*value="\(.*\)".*/\1/p')
+# Make a second request with our session + csrf token to actually log in
+# shellcheck disable=SC2086
+curl $CURLOPTS -c "$COOKIEJAR" -F "_csrf_token=$CSRFTOKEN" -F "_username=admin" -F "_password=$ADMINPASS" "http://localhost/domjudge/login"
+
+# Move back to the default directory
+cd "$DIR"
+
+cp "$COOKIEJAR" cookies.txt
+sed -i 's/#HttpOnly_//g' cookies.txt
+sed -i 's/\t0\t/\t1999999999\t/g' cookies.txt
+section_end
+
+# Could try different entrypoints
+FOUNDERR=0
+URL=public
+mkdir "$URL"
+cd "$URL"
+cp "$DIR"/cookies.txt ./
+section_start "Scrape the site with the rebuild admin user"
+set +e
+wget \
+    --reject-regex logout \
+    --recursive \
+    --no-clobber \
+    --page-requisites \
+    --html-extension \
+    --convert-links \
+    --restrict-file-names=windows \
+    --domains localhost \
+    --no-parent \
+    --load-cookies cookies.txt \
+    http://localhost/domjudge/"$URL"
+set -e
+RET=$?
+section_end
+
+section_start "Archive downloaded site"
+cp -r localhost $ARTIFACTS/
+section_end
+
+section_start "Analyse failures"
+#https://www.gnu.org/software/wget/manual/html_node/Exit-Status.html
+# Exit code 4 is network error which we can ignore
+# Exit code 8 can also be because of HTTP404 or 400
+if [ $RET -ne 4 ] && [ $RET -ne 0 ] && [ $RET -ne 8 ]; then
+    exit $RET
+fi
+
+EXPECTED_HTTP_CODES="200\|302\|400\|404"
+if [ "$ROLE" = "public" ]; then
+    # It's expected to encounter a 401 for the login page as we supply the wrong password
+    EXPECTED_HTTP_CODES="$EXPECTED_HTTP_CODES\|401"
+fi
+set +e
+NUM_ERRORS=$(grep -v "HTTP/1.1\" \($EXPECTED_HTTP_CODES\)" /var/log/nginx/domjudge.log | grep -v "robots.txt" -c; if [ "$?" -gt 1 ]; then exit 127; fi)
+set -e
+echo "$NUM_ERRORS"
+
+if [ "$NUM_ERRORS" -ne 0 ]; then
+    echo "Entered the if"
+    grep -v "HTTP/1.1\" \($EXPECTED_HTTP_CODES\)" /var/log/nginx/domjudge.log | grep -v "robots.txt"
+    exit 1
+fi
+section_end
+
+if [ "$TEST" = "w3cval" ]; then
+    section_start "Remove files from upstream with problems"
+    rm -rf localhost/domjudge/doc
+    rm -rf localhost/domjudge/css/fontawesome-all.min.css*
+    rm -rf localhost/domjudge/bundles/nelmioapidoc*
+    rm -f localhost/domjudge/css/bootstrap.min.css*
+    rm -f localhost/domjudge/css/select2-bootstrap*.css*
+    rm -f localhost/domjudge/css/dataTables*.css*
+    rm -f localhost/domjudge/jury/config/check/phpinfo*
+    section_end
+
+    section_start "Install testsuite"
+    cd "$DIR"
+    wget https://github.com/validator/validator/releases/latest/download/vnu.linux.zip
+    unzip -q vnu.linux.zip
+    section_end
+
+    FLTR='--filterpattern .*autocomplete.*|.*style.*|.*role=tab.*|.*descendant.*|.*Stray.*|.*attribute.*|.*Forbidden.*|.*stream.*'
+    for typ in html css svg
+    do
+        section_start "Analyse with $typ"
+	# shellcheck disable=SC2086
+        "$DIR"/vnu-runtime-image/bin/vnu --errors-only --exit-zero-always --skip-non-$typ --format json $FLTR "$URL" 2> result.json
+	# shellcheck disable=SC2086
+	NEWFOUNDERRORS=$("$DIR"/vnu-runtime-image/bin/vnu --errors-only --exit-zero-always --skip-non-$typ --format gnu $FLTR "$URL" 2>&1 | wc -l)
+        FOUNDERR=$((NEWFOUNDERRORS+FOUNDERR))
+        python3 -m "json.tool" < result.json > "$ARTIFACTS/w3c$typ$URL.json"
+        trace_off; python3 gitlab/jsontogitlab.py "$ARTIFACTS/w3c$typ$URL.json"; trace_on
+        section_end
+    done
+else
+    section_start "Remove files from upstream with problems"
+    rm -rf localhost/domjudge/{doc,api}
+    section_end
+
+    if [ "$TEST" == "axe" ]; then
+        STAN="-e $TEST"
+        FLTR=""
+    else
+        STAN="-s $TEST"
+        FLTR="-E '#DataTables_Table_0 > tbody > tr > td > a','#menuDefault > a','#filter-card > div > div > div > span > span:nth-child(1) > span > ul > li > input',.problem-badge"
+    fi
+    chown -R domjudge:domjudge "$DIR"
+    cd "$DIR"
+    ACCEPTEDERR=5
+    # shellcheck disable=SC2044,SC2035
+    for file in $(find $URL -name "*.html")
+    do
+        section_start "$file"
+        su domjudge -c "pa11y --config .github/jobs/pa11y_config.json $STAN -r json -T $ACCEPTEDERR $FLTR $file" | python3 -m json.tool
+	ERR=$(su domjudge -c "pa11y --config .github/jobs/pa11y_config.json $STAN -r csv -T $ACCEPTEDERR $FLTR $file" | wc -l)
+        FOUNDERR=$((ERR+FOUNDERR-1)) # Remove header row
+        section_end
+    done
+fi
+
+echo "Found: " $FOUNDERR
+[ "$FOUNDERR" -eq 0 ]