From 94ce738163468c1295f1266c65118326b74e4998 Mon Sep 17 00:00:00 2001 From: Rich Megginson Date: Wed, 14 Sep 2022 10:55:09 -0600 Subject: [PATCH 1/3] Many improvements, docs for running tests in containers. See the README e.g. `tox -e container-ansible-core-2.15 -- --image-name centos-9 tests/tests_default.yml` Signed-off-by: Rich Megginson --- README.md | 152 +++++++++- src/tox_lsr/test_scripts/runcontainer.sh | 337 +++++++++++++++++++---- 2 files changed, 418 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 71f8413..0cf9bfe 100644 --- a/README.md +++ b/README.md @@ -391,7 +391,7 @@ arguments you provide. Note that you must use `--` on the command line after the `-e qemu` or `-e qemu-ansible-core-2.x` so that `tox` will not attempt to interpret these as `tox` arguments: ``` -tox -e qemu -- --image-name fedora-34 ... +tox -e qemu -- --image-name fedora-36 ... ``` You must provide one of `--image-file` or `--image-name`. @@ -400,7 +400,7 @@ You must provide one of `--image-file` or `--image-name`. corresponding environment variable is `LSR_QEMU_IMAGE_FILE`. * `--image-name` - assuming you have a config file (`--config`) that maps the given image name to an image url and optional setup, you can just specify an - image name like `--image-name fedora-34` and the script will download the + image name like `--image-name fedora-36` and the script will download the latest qcow2 compose image for Fedora 34 to a local cache (`--cache`). The script will check to see if the downloaded image in the cache is the latest, and will not download if not needed. In the config file you can specify @@ -452,7 +452,7 @@ You must provide one of `--image-file` or `--image-name`. the backing file. This is useful when you want to pre-load a image for testing multiple test runs, but do not want to alter the original downloaded image e.g. pre-configuring package repos, pre-installing packages, etc. This will create - a file called `$IMAGE_PATH.snap` e.g. `~/.cache/linux-system-roles/fedora-34.qcow2.snap`. + a file called `$IMAGE_PATH.snap` e.g. `~/.cache/linux-system-roles/fedora-36.qcow2.snap`. The default is `false`. The corresponding environment variable is `LSR_QEMU_USE_SNAPSHOT`. * `--wait-on-qemu` - This tells the script to wait for qemu to fully exit after @@ -514,19 +514,19 @@ Each additional command line argument is passed through to ansible-playbook, so it must either be an argument or a playbook. If you want to pass both arguments and playbooks, separate them with a `--` on the command line: ``` -tox -e qemu -- --image-name fedora-34 --become --become-user root -- tests_default.yml +tox -e qemu -- --image-name fedora-36 --become --become-user root -- tests_default.yml ``` This is because `runqemu` cannot tell the difference between an Ansible argument and a playbook. If you do not have any ansible-playbook arguments, only playbooks, you can omit the `--`: ``` -tox -e qemu -- --image-name fedora-34 tests_default.yml +tox -e qemu -- --image-name fedora-36 tests_default.yml ``` If using `--collection`, it is assumed you used `tox -e collection` first. Then specify the path to the test playbook inside this collection: ``` tox -e collection -tox -e qemu -- --image-name fedora-34 --collection .tox/ansible_collections/fedora/linux-system-roles/tests/ROLE/tests_default.yml +tox -e qemu -- --image-name fedora-36 --collection .tox/ansible_collections/fedora/linux-system-roles/tests/ROLE/tests_default.yml ``` The config file looks like this: @@ -534,8 +534,8 @@ The config file looks like this: { "images": [ { - "name": "fedora-34", - "compose": "https://kojipkgs.fedoraproject.org/compose/cloud/latest-Fedora-Cloud-34/compose/", + "name": "fedora-36", + "compose": "https://kojipkgs.fedoraproject.org/compose/cloud/latest-Fedora-Cloud-36/compose/", "setup": [ { "name": "Enable HA repos", @@ -556,9 +556,9 @@ The config file looks like this: Example: ``` -tox -e qemu -- --image-name fedora-34 tests/tests_default.yml +tox -e qemu -- --image-name fedora-36 tests/tests_default.yml ``` -This will lookup `fedora-34` in your `~/.config/linux-system-roles.json`, will +This will lookup `fedora-36` in your `~/.config/linux-system-roles.json`, will check if it needs to download a new image to `~/.cache/linux-system-roles`, will create a setup playbook based on the `"setup"` section in the config, and will run `ansible-playbook` with `standard-inventory-qcow2` as the inventory script @@ -705,3 +705,135 @@ python runqemu.py --config=NONE --cache=/path/to/cache_dir \ Older versions of `runqemu.py` would examine the `--image-name` or `--image-file` value to determine if the image is an EL6 image, but this is now deprecated. You must specify `--ssh-el6` if you need it. + +### Container testing + +Integration tests can be run using containers. `tox` will start one or +more containers for the managed nodes using `podman` and the Ansible `podman` connection +plugin. There are some test envs that can be used to run these +tests: +* `container-ansible-2.9` - tests against Ansible 2.9 + * also uses jinja 2.7 if supported by the python used +* `container-ansible-core-2.x` - tests against ansible-core 2.x + +These tests run in one of two modes, depending on which of the following +arguments you provide. Note that you must use `--` on the command line after +the `-e container-ansible-2.9` or `-e container-ansible-core-2.x` so that `tox` will not attempt to +interpret these as `tox` arguments: +``` +tox -e container-ansible-core-2.15 -- --image-name fedora-36 ... +``` +You must provide `--image-name`. + +* `--image-name` - assuming you have a config file (`--config`) that maps the + given image name to a container registry image and optional setup, you can just specify an + image name like `--image-name fedora-38` and the script will pull the + latest image for Fedora 38. In the config file you can specify + additional setup steps to be run e.g. setting up additional dnf/yum repos. + The corresponding environment variable is `CONTAINER_IMAGE_NAME`. +* `--config` - default `$HOME/.config/linux-system-roles.json` - this is the + full path to a config file that lists the image names, the source or compose, + and additional setup steps. The corresponding environment variable is + `CONTAINER_CONFIG`. +* `--erase-old-snapshot` - If given, erase the current snapshot. If not specified + `tox` will use the current snapshot if it is less than + `CONTAINER_AGE` hours old (default: 24) +* `--parallel N` - N is an integer which is the number of containers to run in + parallel. See below. The default is `0` which means playbooks will be run + sequentially. +* `--log-dir PATH` - When using `--parallel` - for each test playbook named `TESTNAME.yml` + the log will be written to a file named `TESTNAME.log` in the `PATH` directory. + The default is the directory where `TESTNAME.yml` is found. +* `--fail-fast true|false` - default `false`, which means all of the tests will be run + and continue through failures - if `true`, the tests will stop running when the first + error is hit. This only applies if you specify multiple playbooks. +Each additional command line argument is passed through to ansible-playbook, so +it must either be an argument or a playbook. If you want to pass both arguments +and playbooks, separate them with a `--` on the command line: +``` +tox -e container-ansible-core-2.15 -- --image-name fedora-38 --become --become-user root -- tests_default.yml +``` +This is because `tox` cannot tell the difference between an Ansible argument +and a playbook. If you do not have any ansible-playbook arguments, only +playbooks, you can omit the `--`: +``` +tox -e container-ansible-core-2.15 -- --image-name fedora-38 tests_default.yml +``` +If you want to test a collection, you must use `tox -e collection` first. Then +specify the path to the test playbook inside this collection: +``` +tox -e collection +tox -e container-ansible-core-2.15 -- --image-name fedora-38 .tox/ansible_collections/fedora/linux-system-roles/tests/ROLE/tests_default.yml +``` + +The config file looks like this: +``` +{ + "images": [ + { + "name": "centos-9", + "centoshtml": "https://cloud.centos.org/centos/9-stream/x86_64/images", + "container": "quay.io/centos/centos:stream9", + "setup": [ + { + "name": "Enable HA repos", + "hosts": "all", + "gather_facts": false, + "tasks": [ + { "name": "Enable HA repos", + "command": "dnf config-manager --set-enabled highavailability" + } + ] + } + ] + }, + ... +} +``` + +Example: +``` +tox -e container-ansible-core-2.15 -- --image-name centos-9 tests/tests_default.yml +``` +This will lookup `centos-9` in your `~/.config/linux-system-roles.json`, check +if the local snapshot is more than `CONTAINER_AGE` hours old, if so will +`podman pull` the `"container"` value, will create a local snapshot container of +this, will create a setup playbook based on the `"setup"` section in the config, +will save the snapshot and will run `ansible-playbook` with `-c podman` using +the local snapshot with `tests/tests_default.yml`. + +The environment variables are useful for customizing in your local tox.ini. For +example, if I want to use a custom location for my config, and I do not +want to use the profile_tasks plugin, I can do this: +``` +[container_common] +setenv = + LSR_QEMU_CONFIG = /home/username/myconfig.json + LSR_QEMU_PROFILE = false +``` + +#### Parallel containers + +By default, if you specify multiple playbooks, each one will be run sequentially +in the same container by the same `ansible-playbook` command e.g. +`ansible-playbook -c podman test1.yml ... testN.yml`. If you use `--parallel M` +with `M > 1` then `tox` will run each playbook separately in separate containers +e.g. +``` +{ podman run -d ...; ansible-playbook -c podman test1.yml; } & +{ podman run -d ...; ansible-playbook -c podman test2.yml; } & +... +{ podman run -d ...; ansible-playbook -c podman testN.yml; } & + +``` +The number `M` controls how many `podman` containers are running at a time. +There seems to be a limitation on the number of containers that can be run at +the same time. For example, if there are 10 test playbooks, and you use +`--parallel 3`, then `tox` will run `3` containers at the same time. As soon as +one completes, it will start another one, until there are `3` running, or there +are no more playbooks to run. + +NOTE: There seems to be a limitation on the number of containers that can be run +in parallel. In testing on my Fedora 36 laptop with 8 processors and lots of +RAM, I can only run `3` at a time - using `4` will cause the containers to fail +with `dbus` errors - not sure what the issue is. diff --git a/src/tox_lsr/test_scripts/runcontainer.sh b/src/tox_lsr/test_scripts/runcontainer.sh index 64474bc..5c1f597 100755 --- a/src/tox_lsr/test_scripts/runcontainer.sh +++ b/src/tox_lsr/test_scripts/runcontainer.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail CONTAINER_OPTS="--privileged --systemd=true --hostname ${CONTAINER_HOSTNAME:-sut}" CONTAINER_MOUNTS="-v /sys/fs/cgroup:/sys/fs/cgroup" @@ -11,14 +11,61 @@ CONTAINER_MOUNTS="-v /sys/fs/cgroup:/sys/fs/cgroup" CONTAINER_AGE=${CONTAINER_AGE:-24} # hours CONTAINER_TESTS_PATH=${CONTAINER_TESTS_PATH:-"$TOXINIDIR/tests"} CONTAINER_SKIP_TAGS=${CONTAINER_SKIP_TAGS:---skip-tags tests::no_container} -CONFIG=$HOME/.config/linux-system-roles.json +CONTAINER_CONFIG="$HOME/.config/linux-system-roles.json" -COLLECTION_BASE_PATH=${COLLECTION_BASE_PATH:-$TOX_WORK_DIR} +COLLECTION_BASE_PATH="${COLLECTION_BASE_PATH:-$TOX_WORK_DIR}" +export ANSIBLE_COLLECTIONS_PATHS="${ANSIBLE_COLLECTIONS_PATHS:-$COLLECTION_BASE_PATH}" +LOCAL_COLLECTION="${LOCAL_COLLECTION:-fedora/linux_system_roles}" + +logit() { + local level + level="$1"; shift + echo ::"$level" "$(date -Isec)" "$@" +} + +info() { + logit info "$@" +} + +notice() { + logit notice "$@" +} + +warning() { + logit warning "$@" +} + +error() { + logit error "$@" +} install_requirements() { - if [ -f meta/requirements.yml ]; then - ansible-galaxy collection install -p "$COLLECTION_BASE_PATH" -vv -r meta/requirements.yml - export ANSIBLE_COLLECTIONS_PATHS="${ANSIBLE_COLLECTIONS_PATHS:-$COLLECTION_BASE_PATH}" + local rq save_tar force update coll_path + # see what capabilities ansible-galaxy has + if ansible-galaxy collection install --help 2>&1 | grep -q -- --force; then + force=--force + fi + if ansible-galaxy collection install --help 2>&1 | grep -q -- --upgrade; then + upgrade=--upgrade + fi + coll_path="$COLLECTION_BASE_PATH/ansible_collections" + if [ -d "$coll_path/$LOCAL_COLLECTION" ]; then + info saving local collection at "$coll_path/$LOCAL_COLLECTION" + save_tar="$(mktemp)" + tar cfP "$save_tar" -C "$coll_path" "$LOCAL_COLLECTION" + trap "rm -f $save_tar" RETURN + fi + for rq in meta/requirements.yml meta/collection-requirements.yml; do + if [ -f "$rq" ]; then + if [ "$rq" = meta/requirements.yml ]; then + warning use meta/collection-requirements.yml instead of "$rq" + fi + ansible-galaxy collection install ${force:-} ${upgrade:-} -p "$COLLECTION_BASE_PATH" -vv -r "$rq" + fi + done + if [ -n "${save_tar:-}" ] && [ -f "${save_tar:-}" ]; then + tar xfP "$save_tar" -C "$coll_path" --overwrite + info restoring local collection at "$coll_path/$LOCAL_COLLECTION" fi } @@ -96,7 +143,7 @@ refresh_test_container() { # shellcheck disable=SC2128 created=$(date --date="$BASH_REMATCH" +%s) else - echo ERROR: invalid date format: "$created" + error invalid date format: "$created" return 1 fi @@ -122,10 +169,12 @@ refresh_test_container() { container_id=$(podman run -d $CONTAINER_OPTS ${LSR_CONTAINER_OPTS:-} \ $CONTAINER_MOUNTS "$image_name" sleep 3600) if [ -z "$container_id" ]; then - echo ERROR: Failed to start container + error Failed to start container return 1 fi + sleep 1 # ensure container is running if ! podman exec -i "$container_id" "$pkgcmd" install -y $initpkgs; then + podman inspect "$container_id" podman rm -f "$container_id" return 1 fi @@ -139,32 +188,38 @@ refresh_test_container() { container_id=$(podman run -d $CONTAINER_OPTS ${LSR_CONTAINER_OPTS:-} \ $CONTAINER_MOUNTS "$image_name" "$CONTAINER_ENTRYPOINT") if [ -z "$container_id" ]; then - echo ERROR: Failed to start container + error Failed to start container return 1 fi + sleep 1 # ensure container is running + inv_file="$(mktemp)" + echo "sut ansible_host=$container_id ansible_connection=podman" > "$inv_file" # shellcheck disable=SC2064 - trap "podman rm -f $container_id" RETURN + trap "podman rm -f $container_id; rm -f $inv_file" RETURN if [ -n "${prepkgs:-}" ]; then if ! podman exec -i "$container_id" "$pkgcmd" install -y $prepkgs; then + podman inspect "$container_id" return 1 fi fi - if [ -n "${setup_yml:-}" ] && [ -f "${setup_yml}" ]; then - if ! ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} -c podman -i "$container_id", \ + if [ -n "${setup_yml:-}" ] && [ -f "${setup_yml}" ] && [ -s "${setup_yml}" ]; then + if ! ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} -i "$inv_file" \ "$setup_yml"; then return 1 fi fi if ! podman exec -i "$container_id" "$pkgcmd" upgrade -y; then + podman inspect "$container_id" return 1 fi COMMON_PKGS="sudo procps-ng systemd-udev device-mapper openssh-server \ - openssh-clients" + openssh-clients iproute" if ! podman exec -i "$container_id" "$pkgcmd" install -y $COMMON_PKGS; then + podman inspect "$container_id" return 1 fi if [ -f "${CONTAINER_TESTS_PATH}/setup-snapshot.yml" ]; then - if ! ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} -c podman -i "$container_id", \ + if ! ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} -i "$inv_file" \ "${CONTAINER_TESTS_PATH}/setup-snapshot.yml"; then return 1 fi @@ -176,7 +231,148 @@ refresh_test_container() { return 0 } +setup_vault() { + local test_dir test_basename no_vault_vars vault_pwd_file vault_vars_file + test_dir="$1" + test_basename="$2" + vault_pwd_file="$test_dir/vault_pwd" + vault_vars_file="$test_dir/vars/vault-variables.yml" + no_vault_vars="$test_dir/no-vault-variables.txt" + if [ -f "$vault_pwd_file" ] && [ -f "$vault_vars_file" ]; then + export ANSIBLE_VAULT_PASSWORD_FILE="$vault_pwd_file" + vault_args="--extra-vars=@$vault_vars_file" + if [ -f "$no_vault_vars" ] && grep -q "^${test_basename}$" "$no_vault_vars"; then + unset ANSIBLE_VAULT_PASSWORD_FILE + vault_args="" + fi + else + unset ANSIBLE_VAULT_PASSWORD_FILE + vault_args="" + fi +} + +run_playbooks() { + # shellcheck disable=SC2086 + local test_pb_base test_dir pb + declare -a test_pb=() + test_pb_base="$1"; shift + for pb in "$@"; do + pb="$(realpath "$pb")" + test_pb+=("$pb") + if [ -z "${test_dir:-}" ]; then + test_dir="$(dirname "$pb")" + fi + done + + container_id=$(podman run -d $CONTAINER_OPTS --name "$test_pb_base" \ + ${LSR_CONTAINER_OPTS:-} $CONTAINER_MOUNTS "$CONTAINER_IMAGE" \ + "$CONTAINER_ENTRYPOINT") + + if [ -z "$container_id" ]; then + error Failed to start container + exit 1 + fi + + inv_file="$(mktemp)" + if [ -z "${LSR_DEBUG:-}" ]; then + trap "rm -rf $inv_file; podman rm -f $container_id" RETURN EXIT + fi + sleep 1 # give the container a chance to start up + if ! podman exec -i "$container_id" /bin/bash -euxo pipefail -c ' + limit=60 + for ii in $(seq 1 $limit); do + if systemctl is-active dbus; then + break + fi + sleep 1 + done + if [ $ii = $limit ]; then + systemctl status dbus + exit 1 + fi + sysctl -w net.ipv6.conf.all.disable_ipv6=0 + systemctl unmask systemd-udevd + if ! systemctl start systemd-udevd; then + systemctl status systemd-udevd + exit 1 + fi + for ii in $(seq 1 $limit); do + if systemctl is-active systemd-udevd; then + break + fi + sleep 1 + done + if [ $ii = $limit ]; then + systemctl status systemd-udevd + exit 1 + fi + '; then + podman inspect "$container_id" + return 1 + fi + + echo "sut ansible_host=$container_id ansible_connection=podman" > "$inv_file" + setup_vault "$test_dir" "${test_pb_base}.yml" + # shellcheck disable=SC2086 + pushd "$test_dir" > /dev/null + ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} -i "$inv_file" ${vault_args:-} \ + -e ansible_playbook_filepath="$(type -p ansible-playbook)" "${test_pb[@]}" + popd > /dev/null +} + +# grr - ubuntu 20.04 bash wait does not support -p pid :-( +# also - you cannot call this function in a subshell or it +# defeats the purpose of finding child jobs +# PID is a global variable +lsr_wait() { + local rc found + while true; do + rc=0 + wait -nf || rc=$? + # now, figure out which pid just exited, if any + # it might not be one of our playbook jobs that exited + found=0 + for PID in "${!cur_jobs[@]}"; do + test -d "/proc/$PID" || { found=1; break; } + done + if [ "$found" = 0 ]; then + sleep 1 + else + break + fi + done + return "$rc" +} + +# PID is a global variable +wait_for_results() { + local rc test_pb_base + rc=0 + info waiting for results - pids "${!cur_jobs[*]}" tests "${cur_jobs[*]}" + for mypid in ${!cur_jobs[*]}; do + test -d "/proc/$mypid" || echo "$mypid" is not running + done + # wait -p pid -n || rc=$? + lsr_wait || rc=$? + test_pb_base="${cur_jobs[$PID]}" + results["$test_pb_base"]="$rc" + unset cur_jobs["$PID"] + if [ "$rc" = 0 ]; then + info Test "$test_pb_base" SUCCESS + else + error Test "$test_pb_base" FAILED + fi + rc=0 + podman wait "$test_pb_base" > /dev/null 2>&1 || rc=$? + # 125 is no such container - ok + if [ "$rc" != 0 ] && [ "$rc" != 125 ]; then + error container "$test_pb_base" in invalid wait state: "$rc" + fi +} + ERASE_OLD_SNAPSHOT=false +PARALLEL=0 +FAIL_FAST=true while [ -n "${1:-}" ]; do key="$1" case "$key" in @@ -185,9 +381,18 @@ while [ -n "${1:-}" ]; do CONTAINER_IMAGE_NAME="$1" ;; --config) shift - CONFIG="$1" ;; + CONTAINER_CONFIG="$1" ;; --erase-old-snapshot) ERASE_OLD_SNAPSHOT=true ;; + --parallel) + shift + PARALLEL="$1" ;; + --fail-fast) + shift + FAIL_FAST="$1" ;; + --log-dir) + shift + LOG_DIR="$1" ;; --*) # unknown option echo "Unknown option $1" exit 1 ;; @@ -197,18 +402,20 @@ while [ -n "${1:-}" ]; do shift done -CONTAINER_BASE_IMAGE=$(jq -r '.images[] | select(.name == "'"$CONTAINER_IMAGE_NAME"'") | .container' "$CONFIG") +CONTAINER_BASE_IMAGE=$(jq -r '.images[] | select(.name == "'"$CONTAINER_IMAGE_NAME"'") | .container' "$CONTAINER_CONFIG") if [ "${CONTAINER_BASE_IMAGE:-null}" = null ] ; then - echo ERROR: container named "$CONTAINER_IMAGE_NAME" not found in "$CONFIG" + error container named "$CONTAINER_IMAGE_NAME" not found in "$CONTAINER_CONFIG" exit 1 fi CONTAINER_IMAGE=${CONTAINER_IMAGE:-"lsr-test-$CONTAINER_IMAGE_NAME:latest"} setup_json=$(mktemp --suffix _setup.json) setup_yml=$(mktemp --suffix _setup.yml) -jq -r '.images[] | select(.name == "'"$CONTAINER_IMAGE_NAME"'") | .setup' "$CONFIG" > "$setup_json" +jq -r '.images[] | select(.name == "'"$CONTAINER_IMAGE_NAME"'") | .setup' "$CONTAINER_CONFIG" > "$setup_json" python -c ' import json, yaml, sys val = json.load(open(sys.argv[1])) +if not val: + sys.exit(0) yaml.safe_dump(val, open(sys.argv[2], "w")) ' "$setup_json" "$setup_yml" rm -f "$setup_json" @@ -219,59 +426,67 @@ if [ -z "${CONTAINER_ENTRYPOINT:-}" ]; then esac fi +echo ::group::install collection requirements install_requirements +echo ::endgroup:: +echo ::group::install and configure plugins used by tests setup_plugins +echo ::endgroup:: +echo ::group::setup and prepare test container if ! refresh_test_container "$ERASE_OLD_SNAPSHOT" "$setup_yml"; then rm -f "$setup_yml" exit 1 fi +echo ::endgroup:: rm -f "$setup_yml" -export TEST_ARTIFACTS="${TEST_ARTIFACTS:-artifacts}" - -# shellcheck disable=SC2086 -CONTAINER_ID=$(podman run -d $CONTAINER_OPTS ${LSR_CONTAINER_OPTS:-} \ - $CONTAINER_MOUNTS "$CONTAINER_IMAGE" "$CONTAINER_ENTRYPOINT") - -if [ -z "$CONTAINER_ID" ]; then - echo ERROR: Failed to start container - exit 1 -fi - -clean_up() { - podman rm -f "$CONTAINER_ID" || true -} - -if [ -z "${DEBUG:-}" ]; then - trap clean_up EXIT -fi - -podman exec -i "$CONTAINER_ID" /bin/bash -euxo pipefail -c ' - limit=30 - for ii in $(seq 1 $limit); do - if systemctl is-active dbus; then - break +declare -A cur_jobs=() +declare -A log_files=() +declare -A results=() +declare -A test_pb=() +if [ "$PARALLEL" -gt 1 ]; then + while [ -n "${1:-}" ]; do + if [ "${#cur_jobs[*]}" -lt "$PARALLEL" ]; then + orig_test_pb="$1"; shift + test_pb="$(realpath "$orig_test_pb")" + test_pb_base="$(basename "$test_pb" .yml)" + log_dir="${LOG_DIR:-$(dirname "$test_pb")}" + log_file="${test_pb_base}.log" + if [ ! -d "$log_dir" ]; then + mkdir -p "$log_dir" + fi + run_playbooks "$test_pb_base" "$test_pb" > "$log_dir/$log_file" 2>&1 & + cur_jobs["$!"]="$test_pb_base" + log_files["$test_pb_base"]="$log_dir/$log_file" + test_pb["$test_pb_base"]="$orig_test_pb" + info Starting test "$test_pb_base" + else + wait_for_results fi - sleep 1 done - if [ $ii = $limit ]; then - systemctl status dbus - exit 1 - fi - sysctl -w net.ipv6.conf.all.disable_ipv6=0 - systemctl unmask systemd-udevd - systemctl start systemd-udevd - for ii in $(seq 1 $limit); do - if systemctl is-active systemd-udevd; then - break - fi - sleep 1 + info All tests executed, waiting for results "${cur_jobs[*]}" + while [ "${#cur_jobs[*]}" -gt 0 ]; do + wait_for_results done - if [ $ii = $limit ]; then - systemctl status systemd-udevd - exit 1 +else + test_pb_base="$(basename "$1" .yml)" + run_playbooks "$test_pb_base" "$@" +fi + +exit_code=0 +for test_pb_base in "${!results[@]}"; do + rc="${results[$test_pb_base]}" + log="${log_files[$test_pb_base]}" + orig_test_pb="${test_pb[$test_pb_base]}" + if [ "$rc" != 0 ]; then + exit_code="$rc" + error "file=${orig_test_pb}::Test" "$test_pb_base FAILED" + else + info "file=${orig_test_pb}::Test" "$test_pb_base PASSED" fi -' +done +if [ "$PARALLEL" -gt 1 ]; then + tar cfz logs.tar.gz "${log_files[@]}" +fi -# shellcheck disable=SC2086 -ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} -c podman -i "$CONTAINER_ID", "$@" +exit "$exit_code" From 6aef92b62f649f7780af5970ce452c5bebf9dedc Mon Sep 17 00:00:00 2001 From: Noriko Hosoi Date: Wed, 28 Jun 2023 14:01:13 -0700 Subject: [PATCH 2/3] Add 2 options to runcontainer.sh (#1) - --extra-rpm : specifies an additional rpm package to install in the container. Default to none. E.g., --extra-rpm diffutils --extra-rpm sudo - --extra-skip-tag : specifies a tag to skip. Default to none. E.g., --extra-skip-tag "tests::use_selinux_role" If no `--parallel` option is given, it runs all the tests in one ansible-playbook command line that is the original behavior. If `--parallel` option is given, it runs each test per command line. To run *.yml per command line in one container, '--parallel 1` option is required. --- src/tox_lsr/test_scripts/runcontainer.sh | 25 ++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/tox_lsr/test_scripts/runcontainer.sh b/src/tox_lsr/test_scripts/runcontainer.sh index 5c1f597..411837f 100755 --- a/src/tox_lsr/test_scripts/runcontainer.sh +++ b/src/tox_lsr/test_scripts/runcontainer.sh @@ -162,6 +162,9 @@ refresh_test_container() { prepkgs="dnf-plugins-core" ;; *) pkgcmd=dnf; prepkgs="" ;; esac + for rpm in ${EXTRA_RPMS:-}; do + initpkgs="$rpm $initpkgs" + done image_name="$CONTAINER_BASE_IMAGE" if [ -n "${initpkgs:-}" ]; then # some images do not have the entrypoint, so that must be installed @@ -315,8 +318,18 @@ run_playbooks() { setup_vault "$test_dir" "${test_pb_base}.yml" # shellcheck disable=SC2086 pushd "$test_dir" > /dev/null - ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} -i "$inv_file" ${vault_args:-} \ - -e ansible_playbook_filepath="$(type -p ansible-playbook)" "${test_pb[@]}" + if [ "$PARALLEL" -gt 0 ]; then + for pb in ${test_pb[@]}; do + ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} ${EXTRA_SKIP_TAGS:-} \ + -i "$inv_file" ${vault_args:-} \ + -e ansible_playbook_filepath="$(type -p ansible-playbook)" "$pb" + done + else + ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} ${EXTRA_SKIP_TAGS:-} \ + -i "$inv_file" ${vault_args:-} \ + -e ansible_playbook_filepath="$(type -p ansible-playbook)" \ + "${test_pb[@]}" + fi popd > /dev/null } @@ -373,6 +386,8 @@ wait_for_results() { ERASE_OLD_SNAPSHOT=false PARALLEL=0 FAIL_FAST=true +EXTRA_RPMS=() +EXTRA_SKIP_TAGS="" while [ -n "${1:-}" ]; do key="$1" case "$key" in @@ -393,6 +408,12 @@ while [ -n "${1:-}" ]; do --log-dir) shift LOG_DIR="$1" ;; + --extra-rpm) + shift + EXTRA_RPMS+=("$1") ;; + --extra-skip-tag) + shift + EXTRA_SKIP_TAGS="--skip-tags $1 $EXTRA_SKIP_TAGS" ;; --*) # unknown option echo "Unknown option $1" exit 1 ;; From 56b9ae2bcc7a88db920bdd546dc48affea705e80 Mon Sep 17 00:00:00 2001 From: Rich Megginson Date: Wed, 28 Jun 2023 15:33:47 -0600 Subject: [PATCH 3/3] fix some coding, shellcheck issues --- README.md | 7 +-- src/tox_lsr/test_scripts/runcontainer.sh | 61 +++++++++++++----------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 0cf9bfe..53446d6 100644 --- a/README.md +++ b/README.md @@ -744,9 +744,10 @@ You must provide `--image-name`. * `--log-dir PATH` - When using `--parallel` - for each test playbook named `TESTNAME.yml` the log will be written to a file named `TESTNAME.log` in the `PATH` directory. The default is the directory where `TESTNAME.yml` is found. -* `--fail-fast true|false` - default `false`, which means all of the tests will be run - and continue through failures - if `true`, the tests will stop running when the first - error is hit. This only applies if you specify multiple playbooks. +* `--extra-rpm` - specifies an additional rpm package to install in the + container. Default to none. E.g., `--extra-rpm diffutils --extra-rpm sudo` +* `--extra-skip-tag` - specifies a tag to skip. Default to none. E.g., + `--extra-skip-tag "tests::some_tag"` Each additional command line argument is passed through to ansible-playbook, so it must either be an argument or a playbook. If you want to pass both arguments and playbooks, separate them with a `--` on the command line: diff --git a/src/tox_lsr/test_scripts/runcontainer.sh b/src/tox_lsr/test_scripts/runcontainer.sh index 411837f..6119efc 100755 --- a/src/tox_lsr/test_scripts/runcontainer.sh +++ b/src/tox_lsr/test_scripts/runcontainer.sh @@ -2,15 +2,15 @@ set -euo pipefail -CONTAINER_OPTS="--privileged --systemd=true --hostname ${CONTAINER_HOSTNAME:-sut}" -CONTAINER_MOUNTS="-v /sys/fs/cgroup:/sys/fs/cgroup" +CONTAINER_OPTS=("--privileged" "--systemd=true" "--hostname" "${CONTAINER_HOSTNAME:-sut}") +CONTAINER_MOUNTS=("-v" "/sys/fs/cgroup:/sys/fs/cgroup") #CONTAINER_ENTRYPOINT="/usr/sbin/init" #CONTAINER_IMAGE_NAME=${CONTAINER_IMAGE_NAME:-centos-8} #CONTAINER_BASE_IMAGE=${CONTAINER_BASE_IMAGE:-quay.io/centos/centos:stream8} #CONTAINER_IMAGE=${CONTAINER_IMAGE:-lsr-test-$CONTAINER_PLATFORM:latest} CONTAINER_AGE=${CONTAINER_AGE:-24} # hours CONTAINER_TESTS_PATH=${CONTAINER_TESTS_PATH:-"$TOXINIDIR/tests"} -CONTAINER_SKIP_TAGS=${CONTAINER_SKIP_TAGS:---skip-tags tests::no_container} +CONTAINER_SKIP_TAGS=${CONTAINER_SKIP_TAGS:---skip-tags tests::uses_selinux} CONTAINER_CONFIG="$HOME/.config/linux-system-roles.json" COLLECTION_BASE_PATH="${COLLECTION_BASE_PATH:-$TOX_WORK_DIR}" @@ -27,9 +27,10 @@ info() { logit info "$@" } -notice() { - logit notice "$@" -} +# not currently used +# notice() { +# logit notice "$@" +# } warning() { logit warning "$@" @@ -40,7 +41,7 @@ error() { } install_requirements() { - local rq save_tar force update coll_path + local rq save_tar force upgrade coll_path # see what capabilities ansible-galaxy has if ansible-galaxy collection install --help 2>&1 | grep -q -- --force; then force=--force @@ -53,6 +54,7 @@ install_requirements() { info saving local collection at "$coll_path/$LOCAL_COLLECTION" save_tar="$(mktemp)" tar cfP "$save_tar" -C "$coll_path" "$LOCAL_COLLECTION" + # shellcheck disable=SC2064 trap "rm -f $save_tar" RETURN fi for rq in meta/requirements.yml meta/collection-requirements.yml; do @@ -60,6 +62,7 @@ install_requirements() { if [ "$rq" = meta/requirements.yml ]; then warning use meta/collection-requirements.yml instead of "$rq" fi + # shellcheck disable=SC2086 ansible-galaxy collection install ${force:-} ${upgrade:-} -p "$COLLECTION_BASE_PATH" -vv -r "$rq" fi done @@ -169,8 +172,8 @@ refresh_test_container() { if [ -n "${initpkgs:-}" ]; then # some images do not have the entrypoint, so that must be installed # first - container_id=$(podman run -d $CONTAINER_OPTS ${LSR_CONTAINER_OPTS:-} \ - $CONTAINER_MOUNTS "$image_name" sleep 3600) + container_id=$(podman run -d "${CONTAINER_OPTS[@]}" ${LSR_CONTAINER_OPTS:-} \ + "${CONTAINER_MOUNTS[@]}" "$image_name" sleep 3600) if [ -z "$container_id" ]; then error Failed to start container return 1 @@ -188,8 +191,8 @@ refresh_test_container() { podman rm -f "$container_id" image_name="$CONTAINER_IMAGE" fi - container_id=$(podman run -d $CONTAINER_OPTS ${LSR_CONTAINER_OPTS:-} \ - $CONTAINER_MOUNTS "$image_name" "$CONTAINER_ENTRYPOINT") + container_id=$(podman run -d "${CONTAINER_OPTS[@]}" ${LSR_CONTAINER_OPTS:-} \ + "${CONTAINER_MOUNTS[@]}" "$image_name" "$CONTAINER_ENTRYPOINT") if [ -z "$container_id" ]; then error Failed to start container return 1 @@ -206,6 +209,7 @@ refresh_test_container() { fi fi if [ -n "${setup_yml:-}" ] && [ -f "${setup_yml}" ] && [ -s "${setup_yml}" ]; then + # shellcheck disable=SC2086 if ! ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} -i "$inv_file" \ "$setup_yml"; then return 1 @@ -222,6 +226,7 @@ refresh_test_container() { return 1 fi if [ -f "${CONTAINER_TESTS_PATH}/setup-snapshot.yml" ]; then + # shellcheck disable=SC2086 if ! ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} -i "$inv_file" \ "${CONTAINER_TESTS_PATH}/setup-snapshot.yml"; then return 1 @@ -256,19 +261,20 @@ setup_vault() { run_playbooks() { # shellcheck disable=SC2086 - local test_pb_base test_dir pb - declare -a test_pb=() + local test_pb_base test_dir pb test_pbs + test_pbs=() test_pb_base="$1"; shift for pb in "$@"; do pb="$(realpath "$pb")" - test_pb+=("$pb") + test_pbs+=("$pb") if [ -z "${test_dir:-}" ]; then test_dir="$(dirname "$pb")" fi done - container_id=$(podman run -d $CONTAINER_OPTS --name "$test_pb_base" \ - ${LSR_CONTAINER_OPTS:-} $CONTAINER_MOUNTS "$CONTAINER_IMAGE" \ + # shellcheck disable=SC2086 + container_id=$(podman run -d "${CONTAINER_OPTS[@]}" --name "$test_pb_base" \ + ${LSR_CONTAINER_OPTS:-} "${CONTAINER_MOUNTS[@]}" "$CONTAINER_IMAGE" \ "$CONTAINER_ENTRYPOINT") if [ -z "$container_id" ]; then @@ -278,6 +284,7 @@ run_playbooks() { inv_file="$(mktemp)" if [ -z "${LSR_DEBUG:-}" ]; then + # shellcheck disable=SC2064 trap "rm -rf $inv_file; podman rm -f $container_id" RETURN EXIT fi sleep 1 # give the container a chance to start up @@ -316,19 +323,20 @@ run_playbooks() { echo "sut ansible_host=$container_id ansible_connection=podman" > "$inv_file" setup_vault "$test_dir" "${test_pb_base}.yml" - # shellcheck disable=SC2086 pushd "$test_dir" > /dev/null if [ "$PARALLEL" -gt 0 ]; then - for pb in ${test_pb[@]}; do + for pb in "${test_pbs[@]}"; do + # shellcheck disable=SC2086 ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} ${EXTRA_SKIP_TAGS:-} \ -i "$inv_file" ${vault_args:-} \ -e ansible_playbook_filepath="$(type -p ansible-playbook)" "$pb" done else + # shellcheck disable=SC2086 ansible-playbook -vv ${CONTAINER_SKIP_TAGS:-} ${EXTRA_SKIP_TAGS:-} \ -i "$inv_file" ${vault_args:-} \ -e ansible_playbook_filepath="$(type -p ansible-playbook)" \ - "${test_pb[@]}" + "${test_pbs[@]}" fi popd > /dev/null } @@ -369,7 +377,7 @@ wait_for_results() { lsr_wait || rc=$? test_pb_base="${cur_jobs[$PID]}" results["$test_pb_base"]="$rc" - unset cur_jobs["$PID"] + unset 'cur_jobs["$PID"]' if [ "$rc" = 0 ]; then info Test "$test_pb_base" SUCCESS else @@ -385,7 +393,6 @@ wait_for_results() { ERASE_OLD_SNAPSHOT=false PARALLEL=0 -FAIL_FAST=true EXTRA_RPMS=() EXTRA_SKIP_TAGS="" while [ -n "${1:-}" ]; do @@ -402,9 +409,6 @@ while [ -n "${1:-}" ]; do --parallel) shift PARALLEL="$1" ;; - --fail-fast) - shift - FAIL_FAST="$1" ;; --log-dir) shift LOG_DIR="$1" ;; @@ -469,14 +473,14 @@ if [ "$PARALLEL" -gt 1 ]; then while [ -n "${1:-}" ]; do if [ "${#cur_jobs[*]}" -lt "$PARALLEL" ]; then orig_test_pb="$1"; shift - test_pb="$(realpath "$orig_test_pb")" - test_pb_base="$(basename "$test_pb" .yml)" - log_dir="${LOG_DIR:-$(dirname "$test_pb")}" + abs_test_pb="$(realpath "$orig_test_pb")" + test_pb_base="$(basename "$abs_test_pb" .yml)" + log_dir="${LOG_DIR:-$(dirname "$abs_test_pb")}" log_file="${test_pb_base}.log" if [ ! -d "$log_dir" ]; then mkdir -p "$log_dir" fi - run_playbooks "$test_pb_base" "$test_pb" > "$log_dir/$log_file" 2>&1 & + run_playbooks "$test_pb_base" "$abs_test_pb" > "$log_dir/$log_file" 2>&1 & cur_jobs["$!"]="$test_pb_base" log_files["$test_pb_base"]="$log_dir/$log_file" test_pb["$test_pb_base"]="$orig_test_pb" @@ -497,7 +501,6 @@ fi exit_code=0 for test_pb_base in "${!results[@]}"; do rc="${results[$test_pb_base]}" - log="${log_files[$test_pb_base]}" orig_test_pb="${test_pb[$test_pb_base]}" if [ "$rc" != 0 ]; then exit_code="$rc"