Enhanced check_schema with additional schema fallbacks

The schema is searched in various places:
- URL given as xs:noNamespaceSchemaLocation in PDSC file
- URL inferred from schemaVersion in PDSC file
- URL of latest schema on GitHub

gen-pack

@@ -1,5 +1,5 @@
 
-GEN_PACK_LIB_VERSION="0.2.2"
+GEN_PACK_LIB_VERSION="0.3.0"
 GEN_PACK_LIB_SOURCE="$(realpath $(dirname ${BASH_SOURCE}))"
 
 shopt -s expand_aliases
@@ -14,6 +14,7 @@ fi
 . "${GEN_PACK_LIB_SOURCE}/lib/utilities"
 . "${GEN_PACK_LIB_SOURCE}/lib/pdsc"
 . "${GEN_PACK_LIB_SOURCE}/lib/gittools"
+. "${GEN_PACK_LIB_SOURCE}/lib/logging"
 
 function add_dirs {
   # Add directories
@@ -76,20 +77,38 @@ function apply_patches {
 
 function check_schema {
   echo "Running schema check for $1"
-  local SCHEMA=$(realpath -m "${CMSIS_TOOLSDIR}/../PACK.xsd")
+  local CMSISSCHEMA=$(realpath -m "${CMSIS_TOOLSDIR:-.}/../PACK.xsd")
+  local SCHEMA="${CMSISSCHEMA}"
   if [ ! -f "$SCHEMA" ]; then
     echo "Fetching schema file..."
     local SCHEMAURL=$(grep -Pio "xs:noNamespaceSchemaLocation=\"\K[^\"]+" "$1")
+    local SCHEMAVERSION=$(grep -Pio "schemaVersion=\"\K[^\"]+" "$1")
     local SCHEMA="${TEMP:-/tmp}/PACK.xsd"
-    echo "\"${UTILITY_CURL}\" -sL $SCHEMAURL --output \"${SCHEMA}\""
-    "${UTILITY_CURL}" -sL $SCHEMAURL --output "${SCHEMA}"
+    curl_download "${SCHEMAURL}" "${SCHEMA}"
     local errorlevel=$?
     if [ $errorlevel -ne 0 ]; then
-      echo "Failed downloading schema from '$SCHEMAURL'. Skipping schema check." >&2
-      exit 1
+        echo "Schema referenced in PDSC file not found. Looking for schema with version '${SCHEMAVERSION}' ..." >&2
+        curl_download "https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/blob/v${SCHEMAVERSION}/schema/PACK.xsd" "${SCHEMA}"
+        errorlevel=$?
+    fi
+    if [ $errorlevel -ne 0 ]; then
+        echo "Schema referenced by PDSC file's schema version not found. Looking for latest schema ..." >&2
+        curl_download "https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/blob/main/schema/PACK.xsd" "${SCHEMA}"
+        errorlevel=$?
+    fi
+    if [ $errorlevel -ne 0 ]; then
+        echo "build aborted: No schema file could be found!" >&2
+        echo " " >&2
+        echo "  Hint: Assure one of the following to get the schema check working." >&2
+        echo "        - Provide PACK.XSD into '${CMSISSCHEMA}'." >&2
+        echo "        - Provide schema URL in PDSC file's xs:noNamespaceSchemaLocation attribute." >&2
+        echo "        - Provide schema release tag version in PDSC file's schemaVersion attribute." >&2        
+        echo "        Available schema releases are listed on https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/tags." >&2        
+        echo " " >&2
+        exit $errorlevel
     fi
   fi
-  echo "\"${UTILITY_XMLLINT}\" --noout --schema \"${SCHEMA}\" \"$1\""
+  echo_v "\"${UTILITY_XMLLINT}\" --noout --schema \"${SCHEMA}\" \"$1\""
   "${UTILITY_XMLLINT}" --noout --schema "${SCHEMA}" "$1"
   local errorlevel=$?
   if [ $errorlevel -ne 0 ]; then
@@ -100,7 +119,7 @@ function check_schema {
 }
 
 function check_pack {
-  echo "\"${UTILITY_PACKCHK}\" \"$1\" -i \"${CMSIS_PACK_ROOT}/.Web/ARM.CMSIS.pdsc\" -n \"${PACK_OUTPUT}/PackName.txt\"" ${PACKCHK_ARGS[@]}
+  echo_v "\"${UTILITY_PACKCHK}\" \"$1\" -i \"${CMSIS_PACK_ROOT}/.Web/ARM.CMSIS.pdsc\" -n \"${PACK_OUTPUT}/PackName.txt\"" ${PACKCHK_ARGS[@]}
   "${UTILITY_PACKCHK}" "$1" -i "${CMSIS_PACK_ROOT}/.Web/ARM.CMSIS.pdsc" -n "${PACK_OUTPUT}/PackName.txt" ${PACKCHK_ARGS[@]}
   local errorlevel=$?
   if [ $errorlevel -ne 0 ]; then

lib/getopts

@@ -22,6 +22,9 @@ function getopts {
       '--no-changelog')
         unset CHANGELOG
       ;;
+      '-v','--verbose')
+        VERBOSE=1
+      ;;
       *)    # unknown option
         POSITIONAL+=("$1") # save it in an array for later
       ;;

lib/logging

@@ -0,0 +1,6 @@
+
+function echo_v {
+  if [[ $VERBOSE == 1 ]]; then 
+    echo $*
+  fi
+}

lib/usage

@@ -1,12 +1,13 @@
 
 function usage {
 cat << EOF
-Usage: $(basename $0) [-h] [-k] [-c <prefix>] [<pdsc>]
+Usage: $(basename $0) [-h] [-k] [-c <prefix>] [-v] [<pdsc>]
 
 Arguments:
   -h, --help                 Print this usage message and exit.
   -k, --keep                 Keep build directory.
-  -c, --changelog <prefix>   Generate changelog. Tags are filterd for <prefix>
+  -c, --changelog <prefix>   Generate changelog. Tags are filtered for <prefix>.
+  -v, --verbose              Print verbose log output.
   <pdsc>                     The pack description to generate the pack for.
 EOF
 }

lib/utilities

@@ -207,7 +207,7 @@ function find_doxygen {
 
 function archive_7zip {
   rm -f "$2"
-  echo "pushd \"$1\"; \"${UTILITY_ZIP}\" a -tzip \"$2\" ."
+  echo_v "pushd \"$1\"; \"${UTILITY_ZIP}\" a -tzip \"$2\" ."
   pushd "$1" 2>&1 >/dev/null
   "${UTILITY_ZIP}" a -tzip "$2" .
   popd 2>&1 >/dev/null
@@ -285,4 +285,19 @@ function check_links {
 
   IFS=$OFS
   return 0
-}
+}
+
+#
+# Usage: curl_download <url> <dest>
+#  <url>   URL to download file from
+#  <dest>  Path to download file to
+function curl_download {
+    echo_v "\"${UTILITY_CURL}\" -sL $1 --output \"$2\""
+    "${UTILITY_CURL}" -sL $1 --output "$2"
+    local errorlevel=$?
+    if [ $errorlevel -ne 0 ]; then
+      echo "Failed downloading file from URL '$1'." >&2
+      return $errorlevel
+    fi
+    return 0
+}

test/tests_gen_pack.sh

@@ -148,11 +148,15 @@ EOF
 
 curl_mock() {
   CURL_MOCK_ARGS="$@"
-  return ${UTILITY_CURL_RESULT:-0}
+  echo "curl_mock $@"
+  local result=${UTILITY_CURL_RESULT[0]:-0}
+  UTILITY_CURL_RESULT=(${UTILITY_CURL_RESULT[@]:1})
+  return $result
 }
 
 xmllint_mock() {
   XMLLINT_MOCK_ARGS="$@"
+  echo "xmllint_mock $@"
   return 0
 }
 
@@ -171,15 +175,62 @@ EOF
   assertContains "${XMLLINT_MOCK_ARGS[@]}" "test.pdsc"
 }
 
-test_check_schema_nourl() {
+test_check_schema_nourl_version() {
   cat > test.pdsc <<EOF
 <?xml version="1.0" encoding="UTF-8"?>
 <package schemaVersion="1.7.7" xmlns:xs="http://www.w3.org/2001/XMLSchema-instance" xs:noNamespaceSchemaLocation="PACK.xsd">
 </package>
 EOF
 
   UTILITY_CURL="curl_mock"
-  UTILITY_CURL_RESULT=6
+  UTILITY_CURL_RESULT=(6 0)
+  UTILITY_XMLLINT="xmllint_mock"
+
+  result=$(check_schema test.pdsc 2>&1)
+  errorlevel=$?
+
+  echo "$result"
+
+  assertEquals 0 $errorlevel
+  assertContains "${result}" "Failed downloading file from URL 'PACK.xsd'."
+  assertContains "${result}" "curl_mock -sL https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/blob/v1.7.7/schema/PACK.xsd"
+  assertNotContains "${result}" "Failed downloading file from URL 'https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/blob/v1.7.7/schema/PACK.xsd'."
+  assertContains "${result}" "xmllint_mock" 
+}
+
+test_check_schema_nourl_main() {
+  cat > test.pdsc <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<package schemaVersion="1.7.7" xmlns:xs="http://www.w3.org/2001/XMLSchema-instance" xs:noNamespaceSchemaLocation="PACK.xsd">
+</package>
+EOF
+
+  UTILITY_CURL="curl_mock"
+  UTILITY_CURL_RESULT=(6 6 0)
+  UTILITY_XMLLINT="xmllint_mock"
+
+  result=$(check_schema test.pdsc 2>&1)
+  errorlevel=$?
+
+  echo "$result"
+
+  assertEquals 0 $errorlevel
+  assertContains "${result}" "Failed downloading file from URL 'PACK.xsd'."
+  assertContains "${result}" "Failed downloading file from URL 'https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/blob/v1.7.7/schema/PACK.xsd'."
+  assertContains "${result}" "curl_mock -sL https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/blob/main/schema/PACK.xsd"
+  assertNotContains "${result}" "Failed downloading file from URL 'https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/blob/main/schema/PACK.xsd'."
+  assertContains "${result}" "xmllint_mock" 
+}
+
+test_check_schema_noschema() {
+  cat > test.pdsc <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<package schemaVersion="1.7.7" xmlns:xs="http://www.w3.org/2001/XMLSchema-instance" xs:noNamespaceSchemaLocation="PACK.xsd">
+</package>
+EOF
+
+  UTILITY_CURL="curl_mock"
+  UTILITY_CURL_RESULT=(6 6 6)
   UTILITY_XMLLINT="xmllint_mock"
 
   result=$(check_schema test.pdsc 2>&1)
@@ -188,7 +239,9 @@ EOF
   echo "$result"
 
   assertNotEquals 0 $errorlevel
-  assertContains "${result}" "Failed downloading schema from 'PACK.xsd'"
+  assertContains "${result}" "Failed downloading file from URL 'PACK.xsd'."
+  assertContains "${result}" "Failed downloading file from URL 'https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/blob/v1.7.7/schema/PACK.xsd'."
+  assertContains "${result}" "Failed downloading file from URL 'https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/blob/main/schema/PACK.xsd'."
   assertNotContains "${result}" "xmllint_mock"
 }
 

test/tests_utilities.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+. "$(dirname "$0")/../lib/logging"
 . "$(dirname "$0")/../lib/utilities"
 
 find() {
@@ -24,6 +25,7 @@ find() {
 }
 
 setUp() {
+  VERBOSE=1
   TESTDIR="${SHUNIT_TMPDIR}/${_shunit_test_}"
   mkdir -p "${TESTDIR}"
   pushd "${TESTDIR}" >/dev/null