Merge pull request #7888 from infosiftr/windows-tests

Adjust test harness to run many tests successfully on Windows

Author: yosifkit

test/clean.sh

@@ -1,4 +1,12 @@
-#!/bin/bash
-set -e
+#!/usr/bin/env bash
+set -Eeuo pipefail
 
-docker images 'librarytest/*' | awk 'NR>1 { print $1":"$2 }' | xargs -r docker rmi
+docker image ls --digests --no-trunc --format '
+	{{- if ne .Tag "<none>" -}}
+		{{- .Repository -}} : {{- .Tag -}}
+	{{- else if ne .Digest "<none>" -}}
+		{{- .Repository -}} @ {{- .Digest -}}
+	{{- else -}}
+		{{- .ID -}}
+	{{- end -}}
+' 'librarytest/*' | xargs -rt docker image rm

test/config.sh

@@ -11,8 +11,6 @@ globalTests+=(
 # for "explicit" images, only run tests that are explicitly specified for that image/variant
 explicitTests+=(
 	[:onbuild]=1
-	[:nanoserver]=1
-	[:windowsservercore]=1
 )
 imageTests[:onbuild]+='
 	override-cmd
@@ -272,6 +270,14 @@ globalExcludeTests+=(
 	[swarm_utc]=1
 	[traefik_utc]=1
 
+	# windows!
+	[:nanoserver_cve-2014--shellshock]=1
+	[:nanoserver_no-hard-coded-passwords]=1
+	[:nanoserver_utc]=1
+	[:windowsservercore_cve-2014--shellshock]=1
+	[:windowsservercore_no-hard-coded-passwords]=1
+	[:windowsservercore_utc]=1
+
 	[hello-world_no-hard-coded-passwords]=1
 	[nats_no-hard-coded-passwords]=1
 	[nats-streaming_no-hard-coded-passwords]=1
@@ -282,9 +288,10 @@ globalExcludeTests+=(
 	# https://github.com/docker-library/official-images/pull/1721#issuecomment-234128477
 	[clearlinux_no-hard-coded-passwords]=1
 
-	# alpine/slim openjdk images are headless and so can't do font stuff
+	# alpine/slim/nanoserver openjdk images are headless and so can't do font stuff
 	[openjdk:alpine_java-uimanager-font]=1
 	[openjdk:slim_java-uimanager-font]=1
+	[openjdk:nanoserver_java-uimanager-font]=1
 	# and adoptopenjdk has opted not to
 	[adoptopenjdk_java-uimanager-font]=1
 
@@ -299,4 +306,10 @@ globalExcludeTests+=(
 	# the Swift slim images are not expected to be able to run the swift-hello-world test because it involves compiling Swift code. The slim images are for running an already built binary.
 	# https://github.com/docker-library/official-images/pull/6302#issuecomment-512181863
 	[swift:slim_swift-hello-world]=1
+
+	# TODO adjust MongoDB tests to use docker networks instead of links so they can work on Windows (and consider using PowerShell to generate appropriate certificates for TLS tests instead of openssl)
+	[mongo:windowsservercore_mongo-basics]=1
+	[mongo:windowsservercore_mongo-auth-basics]=1
+	[mongo:windowsservercore_mongo-tls-basics]=1
+	[mongo:windowsservercore_mongo-tls-auth]=1
 )

test/run.sh

@@ -1,5 +1,5 @@
-#!/bin/bash
-set -e
+#!/usr/bin/env bash
+set -Eeuo pipefail
 
 dir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
 
@@ -46,7 +46,7 @@ while true; do
 	esac
 done
 
-if [ $# -eq 0 ]; then
+if [ "$#" -eq 0 ]; then
 	usage >&2
 	exit 1
 fi
@@ -59,10 +59,21 @@ declare -A globalExcludeTests=()
 declare -A explicitTests=()
 
 # if there are no user-specified configs, use the default config
-if [ ${#configs} -eq 0 ]; then
-	configs+=("$dir/config.sh")
+if [ "${#configs[@]}" -eq 0 ]; then
+	configs+=( "$dir/config.sh" )
 fi
 
+case "$(uname -o)" in
+	Msys)
+		# https://stackoverflow.com/a/34386471/433558
+		# https://github.com/docker/toolbox/blob/6960f28d5b01857d69b2095a02949264f09d3e57/windows/start.sh#L104-L107
+		docker() {
+			MSYS_NO_PATHCONV=1 command docker "$@"
+		}
+		export -f docker
+		;;
+esac
+
 # load the configs
 declare -A testPaths=()
 for conf in "${configs[@]}"; do
@@ -72,7 +83,7 @@ for conf in "${configs[@]}"; do
 	confDir="$(dirname "$conf")"
 
 	for testName in ${globalTests[@]} ${imageTests[@]}; do
-		[ "${testPaths[$testName]}" ] && continue
+		[ -n "${testPaths[$testName]:-}" ] && continue
 
 		if [ -d "$confDir/tests/$testName" ]; then
 			# Test directory found relative to the conf file
@@ -116,11 +127,11 @@ for dockerImage in "$@"; do
 			variant='psmdb'
 			;;
 		*nanoserver*)
-			# all nanoserver variants are windows and should have explict tests
+			# all nanoserver variants are windows
 			variant='nanoserver'
 			;;
 		*windowsservercore*)
-			# all servercore variants are windows and should have explict tests
+			# all servercore variants are windows
 			variant='windowsservercore'
 			;;
 	esac
@@ -131,10 +142,10 @@ for dockerImage in "$@"; do
 	fi
 
 	for possibleAlias in \
-		"${testAlias[$repo:$variant]}" \
-		"${testAlias[$repo]}" \
-		"${testAlias[$testRepo:$variant]}" \
-		"${testAlias[$testRepo]}" \
+		"${testAlias[$repo:$variant]:-}" \
+		"${testAlias[$repo]:-}" \
+		"${testAlias[$testRepo:$variant]:-}" \
+		"${testAlias[$testRepo]:-}" \
 	; do
 		if [ -n "$possibleAlias" ]; then
 			testRepo="$possibleAlias"
@@ -144,9 +155,9 @@ for dockerImage in "$@"; do
 
 	explicitVariant=
 	if [ \
-		"${explicitTests[:$variant]}" \
-		-o "${explicitTests[$repo:$variant]}" \
-		-o "${explicitTests[$testRepo:$variant]}" \
+		"${explicitTests[:$variant]:-}" \
+		-o "${explicitTests[$repo:$variant]:-}" \
+		-o "${explicitTests[$testRepo:$variant]:-}" \
 	]; then
 		explicitVariant=1
 	fi
@@ -156,41 +167,41 @@ for dockerImage in "$@"; do
 		testCandidates+=( "${globalTests[@]}" )
 	fi
 	testCandidates+=(
-		${imageTests[:$variant]}
+		${imageTests[:$variant]:-}
 	)
 	if [ -z "$explicitVariant" ]; then
 		testCandidates+=(
-			${imageTests[$testRepo]}
+			${imageTests[$testRepo]:-}
 		)
 	fi
 	testCandidates+=(
-		${imageTests[$testRepo:$variant]}
+		${imageTests[$testRepo:$variant]:-}
 	)
 	if [ "$testRepo" != "$repo" ]; then
 		if [ -z "$explicitVariant" ]; then
 			testCandidates+=(
-				${imageTests[$repo]}
+				${imageTests[$repo]:-}
 			)
 		fi
 		testCandidates+=(
-			${imageTests[$repo:$variant]}
+			${imageTests[$repo:$variant]:-}
 		)
 	fi
 
 	tests=()
 	for t in "${testCandidates[@]}"; do
-		if [ ${#argTests[@]} -gt 0 -a -z "${argTests[$t]}" ]; then
+		if [ "${#argTests[@]}" -gt 0 ] && [ -z "${argTests[$t]:-}" ]; then
 			# skipping due to -t
 			continue
 		fi
 
 		if [ \
-			! -z "${globalExcludeTests[${testRepo}_$t]}" \
-			-o ! -z "${globalExcludeTests[${testRepo}:${variant}_$t]}" \
-			-o ! -z "${globalExcludeTests[:${variant}_$t]}" \
-			-o ! -z "${globalExcludeTests[${repo}_$t]}" \
-			-o ! -z "${globalExcludeTests[${repo}:${variant}_$t]}" \
-			-o ! -z "${globalExcludeTests[:${variant}_$t]}" \
+			! -z "${globalExcludeTests[${testRepo}_$t]:-}" \
+			-o ! -z "${globalExcludeTests[${testRepo}:${variant}_$t]:-}" \
+			-o ! -z "${globalExcludeTests[:${variant}_$t]:-}" \
+			-o ! -z "${globalExcludeTests[${repo}_$t]:-}" \
+			-o ! -z "${globalExcludeTests[${repo}:${variant}_$t]:-}" \
+			-o ! -z "${globalExcludeTests[:${variant}_$t]:-}" \
 		]; then
 			# skipping due to exclude
 			continue
@@ -223,10 +234,11 @@ for dockerImage in "$@"; do
 		scriptDir="${testPaths[$t]}"
 		if [ -d "$scriptDir" ]; then
 			script="$scriptDir/run.sh"
-			if [ -x "$script" -a ! -d "$script" ]; then
+			if [ -x "$script" ] && [ ! -d "$script" ]; then
 				# TODO dryRun logic
-				if output="$("$script" $dockerImage)"; then
-					if [ -f "$scriptDir/expected-std-out.txt" ] && ! d="$(echo "$output" | diff -u "$scriptDir/expected-std-out.txt" - 2>/dev/null)"; then
+				if output="$("$script" "$dockerImage")"; then
+					output="$(tr -d '\r' <<<"$output")" # Windows gives us \r\n ...  :D
+					if [ -f "$scriptDir/expected-std-out.txt" ] && ! d="$(diff -u "$scriptDir/expected-std-out.txt" - <<<"$output" 2>/dev/null)"; then
 						echo 'failed; unexpected output:'
 						echo "$d"
 						didFail=1
@@ -252,6 +264,6 @@ for dockerImage in "$@"; do
 	done
 done
 
-if [ "$didFail" ]; then
+if [ -n "$didFail" ]; then
 	exit 1
 fi

test/tests/docker-build.sh

@@ -30,4 +30,4 @@ fi
 
 cp -RL "$dir" "$tmp/dir"
 
-docker build -t "$imageTag" "$tmp" > /dev/null
+command docker build -t "$imageTag" "$tmp" > /dev/null

test/tests/hylang-sh/container.hy

@@ -4,8 +4,14 @@
 
 (import subprocess sys)
 (subprocess.check_call [sys.executable "-m" "pip" "install" "-q" "sh"])
+(import platform)
+
+(comment Windows is not supported by sh (sad day))
+(comment https://github.com/amoffat/sh/blob/608f4c3bf5ad75ad40035d03a9c5ffcce0898f07/sh.py#L33-L36)
+(if (= (.system platform) "Windows")
+  (defn echo [dashn num] (return num))
+  (import [sh [echo]]))
 
-(import [sh [echo]])
 (->
   (+
     (int (echo "-n" 21))

test/tests/mongo-basics/run.sh

@@ -22,31 +22,33 @@ if docker info --format '{{ join .SecurityOptions "\n" }}' 2>/dev/null |tac|tac|
 
 	# need set_mempolicy syscall to be able to do numactl for mongodb
 	# if "set_mempolicy" is not in the always allowed list, add it
-	extraSeccomp="$(echo "$seccomp" | docker run -i --rm "$jqImage" --tab '
-		.syscalls[] |= if (
-			.action == "SCMP_ACT_ALLOW"
-			and .args == []
-			and .comment == ""
-			and .includes == {}
-			and .excludes == {}
-		) then (
-			if ( .names | index("set_mempolicy") ) > 0 then
+	extraSeccomp="$(
+		docker run -i --rm "$jqImage" --tab '
+			.syscalls[] |= if (
+				.action == "SCMP_ACT_ALLOW"
+				and .args == []
+				and .comment == ""
+				and .includes == {}
+				and .excludes == {}
+			) then (
+				if ( .names | index("set_mempolicy") ) > 0 then
+					.
+				else (
+					.names |= . + ["set_mempolicy"]
+				) end
+			)
+			else
 				.
-			else (
-				.names |= . + ["set_mempolicy"]
-			) end
-		)
-		else
-			.
-		end
-	')"
+			end
+		' <<<"$seccomp"
+	)"
 else
 	echo >&2 'warning: the current Docker daemon does not appear to support seccomp'
 fi
 
 docker_run_seccomp() {
 	if [ "$haveSeccomp" ]; then
-		docker run --security-opt seccomp=<(echo "$extraSeccomp") "$@"
+		docker run --security-opt seccomp=<(cat <<<"$extraSeccomp") "$@"
 	else
 		docker run "$@"
 	fi

test/tests/override-cmd/run.sh

@@ -1,5 +1,5 @@
-#!/bin/bash
-set -eo pipefail
+#!/usr/bin/env bash
+set -Eeuo pipefail
 
 image="$1"
 
@@ -8,13 +8,22 @@ image="$1"
 
 hello="world-$RANDOM-$RANDOM"
 
+cmd=( echo "Hello $hello" )
+case "$image" in
+	*windowsservercore* | *nanoserver*)
+		cmd=( cmd /Q /S /C "${cmd[*]}" )
+		;;
+esac
+
 # test first with --entrypoint to verify that we even have echo (tests for single-binary images FROM scratch, essentially)
-if ! testOutput="$(docker run --rm --entrypoint echo "$image" "Hello $hello" 2>/dev/null)"; then
+if ! testOutput="$(docker run --rm --entrypoint "${cmd[0]}" "$image" "${cmd[@]:1}" 2>/dev/null)"; then
 	echo >&2 'image does not appear to contain "echo" -- assuming single-binary image'
 	exit
 fi
+testOutput="$(tr -d '\r' <<<"$testOutput")" # Windows gives us \r\n ...  :D
 [ "$testOutput" = "Hello $hello" ]
 
 # now test with normal command to verify the default entrypoint is OK
-output="$(docker run --rm "$image" echo "Hello $hello")"
+output="$(docker run --rm "$image" "${cmd[@]}")"
+output="$(tr -d '\r' <<<"$output")" # Windows gives us \r\n ...  :D
 [ "$output" = "Hello $hello" ]

test/tests/python-hy/container.cmd

@@ -0,0 +1,28 @@
+@echo off
+
+for %%p in ( pypy3 pypy python3 python ) do (
+	%%p --version >nul 2>&1 && (
+		set python=%%p
+		goto found
+	)
+)
+echo unable to run Hy test -- seems this image does not contain Python? >&2
+exit /b 1
+
+:found
+%python% --version >nul 2>&1 || exit /b %errorlevel%
+
+rem Hy is complicated, and uses Python's internal AST representation directly, and thus Hy releases usually lag behind a little on major Python releases (and we don't want that to gum up our tests)
+rem see https://github.com/hylang/hy/issues/1111 for example breakage
+%python% -c "import sys; exit((sys.version_info[0] == 3 and sys.version_info[1] >= 8) or sys.version_info[0] > 3)" || (
+	echo skipping Hy test -- not allowed on Python 3.8+ ^(yet!^) >&2
+	rem cheaters gunna cheat
+	type expected-std-out.txt
+	exit /b 0
+)
+
+pip install -q "hy==0.17.0" || exit /b %errorlevel%
+
+hy ./container.hy || exit /b %errorlevel%
+
+exit /b 0

test/tests/python-hy/container.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-set -e
+set -eu
 
 python=
 for c in pypy3 pypy python3 python; do
@@ -18,9 +18,11 @@ fi
 # see https://github.com/hylang/hy/issues/1111 for example breakage
 if ! "$python" -c 'import sys; exit((sys.version_info[0] == 3 and sys.version_info[1] >= 8) or sys.version_info[0] > 3)'; then
 	echo >&2 'skipping Hy test -- not allowed on Python 3.8+ (yet!)'
-	cat expected-std-out.txt # cheaters gunna cheat
+	# cheaters gunna cheat
+	cat expected-std-out.txt
 	exit
 fi
 
-pip install -q 'hy==0.16.0'
+pip install -q 'hy==0.17.0'
+
 hy ./container.hy