Skip to content

Commit

Permalink
make conformance test more flexible
Browse files Browse the repository at this point in the history
  • Loading branch information
mr-c committed Dec 5, 2023
1 parent 65ddfc9 commit d2059c7
Showing 1 changed file with 37 additions and 25 deletions.
62 changes: 37 additions & 25 deletions conformance-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ set -x
# The directory where this script resides
SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

TMP_DIR=${TMP_DIR:-${SCRIPT_DIRECTORY}}

# Download archive from GitHub
if [[ "${VERSION}" = "v1.0" ]] ; then
REPO=common-workflow-language
Expand All @@ -49,38 +51,40 @@ else
REPO=cwl-$(echo "$VERSION" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/')
fi

if [ ! -d "${REPO}-${GIT_TARGET}" ] ; then
if [ ! -f "${GIT_TARGET}.tar.gz" ] ; then
wget "https://github.com/common-workflow-language/${REPO}/archive/${GIT_TARGET}.tar.gz"
if [ ! -d "${TMP_DIR}/${REPO}-${GIT_TARGET}" ] ; then
if [ ! -f "${TMP_DIR}/${GIT_TARGET}.tar.gz" ] ; then
wget --directory-prefix "${TMP_DIR}" "https://github.com/common-workflow-language/${REPO}/archive/${GIT_TARGET}.tar.gz"
fi
tar xzf "${GIT_TARGET}.tar.gz"
tar --directory "${TMP_DIR}" -x -f "${TMP_DIR}/${GIT_TARGET}.tar.gz"
fi

if [ "${CONTAINER}" == "docker" ]; then
if [ -v SKIP_PULL ] ; then
echo Skipping node container pull.
elif [ "${CONTAINER}" == "docker" ]; then
docker pull docker.io/node:slim
fi

if [ "${CONTAINER}" == "podman" ]; then
elif [ "${CONTAINER}" == "podman" ]; then
podman pull docker.io/node:slim
fi

if [ "${CONTAINER}" == "singularity" ]; then
elif [ "${CONTAINER}" == "singularity" ]; then
export CWL_SINGULARITY_CACHE="$SCRIPT_DIRECTORY/sifcache"
mkdir --parents "${CWL_SINGULARITY_CACHE}"
fi

# Setup environment
venv cwl-conformance-venv
pip install -U setuptools wheel pip
pip uninstall -y cwltool
pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
pip install 'cwltest>=2.3' pytest-cov pytest-xdist
if [ -v SKIP_INSTALL ] ; then
echo 'Skip installing dependencies; cwltool & cwltest must already be installed'
else
venv "${TMP_DIR}/cwl-conformance-venv"
pip install -U setuptools wheel pip
pip uninstall -y cwltool
pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
pip install 'cwltest>=2.3' pytest-cov pytest-xdist
fi

# Set conformance test filename
if [[ "${VERSION}" = "v1.0" ]] ; then
CONFORMANCE_TEST="${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}/${VERSION}/conformance_test_v1.0.yaml"
CONFORMANCE_TEST="${TMP_DIR}/${REPO}-${GIT_TARGET}/${VERSION}/conformance_test_v1.0.yaml"
else
CONFORMANCE_TEST="${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}/conformance_tests.yaml"
CONFORMANCE_TEST="${TMP_DIR}/${REPO}-${GIT_TARGET}/conformance_tests.yaml"
fi
cp "${CONFORMANCE_TEST}" "${CONFORMANCE_TEST%".yaml"}.cwltest.yaml"
CONFORMANCE_TEST="${CONFORMANCE_TEST%".yaml"}.cwltest.yaml"
Expand Down Expand Up @@ -116,14 +120,18 @@ if (( "${#exclusions[*]}" > 0 )); then
fi

# Build command
TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n auto -rs --junit-xml=${SCRIPT_DIRECTORY}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n auto -rs --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
if [[ -n "${EXCLUDE}" ]] ; then
TEST_COMMAND="${TEST_COMMAND} --cwl-exclude ${EXCLUDE}"
fi
TEST_COMMAND="${TEST_COMMAND} --cov --cov-config ${SCRIPT_DIRECTORY}/.coveragerc --cov-report= ${PYTEST_EXTRA}"
if [ -v SKIP_COV ] ; then
echo Skipping gathering of coverage information
else
TEST_COMMAND="${TEST_COMMAND} --cov --cov-config ${SCRIPT_DIRECTORY}/.coveragerc --cov-report= ${PYTEST_EXTRA}"
# Clean up all old coverage data
find "${SCRIPT_DIRECTORY}" \( -type f -name .coverage -or -name '.coverage.*' -or -name coverage.xml \) -delete
fi

# Clean up all old coverage data
find "${SCRIPT_DIRECTORY}" \( -type f -name .coverage -or -name '.coverage.*' -or -name coverage.xml \) -delete

if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$VERSION" = "v1.0" ]] && [[ "$CONTAINER" = "docker" ]]
then
Expand Down Expand Up @@ -153,11 +161,11 @@ echo CWLTOOL_OPTIONS="${CWLTOOL_OPTIONS}"

# Run test
cp "${SCRIPT_DIRECTORY}/tests/cwl-conformance/cwltool-conftest.py" "$(dirname "${CONFORMANCE_TEST}")/conftest.py"
bash -c "${TEST_COMMAND}"
bash -c "cd ${TMP_DIR} && ${TEST_COMMAND}"
RETURN_CODE=$?

# Coverage report
if [ "${RETURN_CODE}" -eq "0" ] ; then
if [ ! -v SKIP_COV ] && [ "${RETURN_CODE}" -eq "0" ] ; then
coverage report
coverage xml
fi
Expand All @@ -172,7 +180,11 @@ then
fi

# Cleanup
deactivate
if [ -z "$SKIP_INSTALL" ] ; then
echo Skipping venv cleanup
else
deactivate
fi
#rm -rf "${GIT_TARGET}.tar.gz" "${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}" "${SCRIPT_DIRECTORY}/cwl-conformance-venv"

# Exit
Expand Down

0 comments on commit d2059c7

Please sign in to comment.