From e6ea468daf466c45149a4ebf01934208ebc865da Mon Sep 17 00:00:00 2001 From: david-waltermire-nist Date: Mon, 27 Jan 2020 22:00:26 -0500 Subject: [PATCH] Completed first pass through CI/CD scripts in support of #596 --- build/ci-cd/config/content | 3 +- build/ci-cd/config/metaschema | 1 + build/ci-cd/copy-and-convert-content.sh | 327 +++++++++--------- build/ci-cd/generate-content-converters.sh | 68 ++-- build/ci-cd/generate-model-documentation.sh | 13 +- build/ci-cd/generate-schema.sh | 13 +- .../include/convert-and-validate-content.sh | 81 +++++ build/ci-cd/include/init-oscal.sh | 7 +- build/ci-cd/package-release.sh | 5 +- build/ci-cd/run-all.sh | 7 +- build/ci-cd/run-unittests.sh | 11 +- ...validate-content-conversion-round-trips.sh | 240 ++++++------- build/ci-cd/validate-content.sh | 8 +- build/ci-cd/validate-metaschema.sh | 10 +- 14 files changed, 445 insertions(+), 349 deletions(-) create mode 100644 build/ci-cd/include/convert-and-validate-content.sh diff --git a/build/ci-cd/config/content b/build/ci-cd/config/content index b46b5cbc41..3360966555 100644 --- a/build/ci-cd/config/content +++ b/build/ci-cd/config/content @@ -1,6 +1,7 @@ # path to source|format of source|model of source|format(s) to convert to src/content/ssp-example/ssp-example.json|json|ssp|xml -src/content/fedramp.gov/xml/*catalog.xml|xml|catalog|json +src/content/fedramp.gov/xml/FedRAMP_catalog.xml|xml|catalog|json src/content/fedramp.gov/xml/*profile.xml|xml|profile|json src/content/nist.gov/SP800-53/rev4/xml/*catalog.xml|xml|catalog|json src/content/nist.gov/SP800-53/rev4/xml/*profile.xml|xml|profile|json + diff --git a/build/ci-cd/config/metaschema b/build/ci-cd/config/metaschema index ae593265a6..c2a164982e 100644 --- a/build/ci-cd/config/metaschema +++ b/build/ci-cd/config/metaschema @@ -3,3 +3,4 @@ src/metaschema/oscal_catalog_metaschema.xml|xml,json|xml,json|xml,json src/metaschema/oscal_profile_metaschema.xml|xml,json|xml,json|xml,json src/metaschema/oscal_component_metaschema.xml|xml,json|xml,json|xml,json src/metaschema/oscal_ssp_metaschema.xml|xml,json|xml,json|xml,json + diff --git a/build/ci-cd/copy-and-convert-content.sh b/build/ci-cd/copy-and-convert-content.sh index fe6109266a..3da85995c2 100755 --- a/build/ci-cd/copy-and-convert-content.sh +++ b/build/ci-cd/copy-and-convert-content.sh @@ -1,16 +1,14 @@ #!/bin/bash -if [[ -z "$OSCALDIR" ]]; then - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - source "$DIR/include/common-environment.sh" + +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh" fi -source "$OSCALDIR/build/ci-cd/include/saxon-init.sh" -source "$OSCALDIR/build/ci-cd/include/init-validate-json.sh" + +source "$OSCALDIR/build/metaschema/scripts/include/init-validate-content.sh" +source "$OSCALDIR/build/ci-cd/include/convert-and-validate-content.sh" # Option defaults -WORKING_DIR="${OSCALDIR}" -VERBOSE=false -HELP=false usage() { # Function: Print a help message. cat << EOF @@ -19,8 +17,6 @@ Usage: $0 [options] -h, --help Display help -w DIR, --working-dir DIR Generate artifacts in DIR -v Provide verbose output ---keep-temp-scratch-dir If a scratch directory is automatically - created, it will not be automatically removed. EOF } @@ -72,164 +68,177 @@ if [ $? -ne 0 ]; then exit 1 fi -exitcode=0 -shopt -s nullglob -shopt -s globstar -while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do +IFS_OLD="$IFS" +while IFS="|" read path_glob format model converttoformats || [[ -n "$path_glob" ]]; do shopt -s extglob - [[ "$path" =~ ^[[:space:]]*# ]] && continue + [[ "$path_glob" =~ ^[[:space:]]*# ]] && continue # remove leading space - path="${path##+([[:space:]])}" + path_glo="${path_glob##+([[:space:]])}" # remove trailing space converttoformats="${converttoformats%%+([[:space:]])}" shopt -u extglob - if [[ ! -z "$path" ]]; then - files_to_process="$OSCALDIR"/"$path" - IFS= # disable word splitting - for file in $files_to_process - do - file_relative=$(realpath --relative-to="${OSCALDIR}" "$file") - dest="$WORKING_DIR/${file/$OSCALDIR\/src\//}" - dest_dir=${dest%/*} # remove filename - mkdir -p "$dest_dir" - dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest") - result=$(cp "$file" "$dest" 2>&1) - cmd_exitcode=$? - if [ $cmd_exitcode -ne 0 ]; then - echo -e "${P_ERROR}Unable to copy '${P_END}${file_relative}${P_ERROR}' to '${P_END}${dest_relative}${P_ERROR}'.${P_END}" + [ -z "$path_glob" ] && continue; + + path_absolute="$OSCALDIR"/"$path_glob" + + for path in $path_absolute; do +# echo "Path: $path" +# echo "Format: $format" +# echo "Model: $model" +# echo "Convert to: $converttoformats" + + paths+=("$path") + formats+=("$format") + models+=("$model") + conversion_formats+=("$converttoformats") + done +done < "${OSCALDIR}/build/ci-cd/config/content" +IFS="$IFS_OLD" + +#echo "Paths: ${paths[@]}" +#echo "Formats: ${formats[@]}" +#echo "Models: ${models[@]}" +#echo "Convert To: ${conversion_formats[@]}" + +shopt -s nullglob +shopt -s globstar + +exitcode=0 +for i in ${!paths[@]}; do +# echo "Index: $i" + source_file="${paths[$i]}" + source_format="${formats[$i]}" + model="${models[$i]}" + converttoformats="${conversion_formats[$i]}" + + # get the base file name + source_file_basename=$(basename $source_file) + source_file_relative=$(get_rel_path "$OSCALDIR" "$source_file") + + # debuggging statements, shows what is processing +# printf 'path: %s\n' "$file" +# printf 'file name: %s\n' "$file_basename" +# printf 'Source format: %s\n' "$source_format" +# printf 'model: %s\n' "$model" +# printf 'convert-to: %s\n' "$converttoformats" + + source_schema="$WORKING_DIR/$source_format/schema/oscal_${model}_schema.xsd" + + # copy source to destination + # -------------------------- + target_file="$WORKING_DIR/${source_file/$OSCALDIR\/src\//}" + target_dir=${target_file%/*} # remove filename + mkdir -p "$target_dir" + target_file_relative=$(get_rel_path "${WORKING_DIR}" "$target_file") + + result=$(cp "$source_file" "$target_file" 2>&1) + cmd_exitcode=$? + if [ $cmd_exitcode -ne 0 ]; then + echo -e "${P_ERROR}Unable to copy '${P_END}${source_file_relative}${P_ERROR}' to '${P_END}${target_file_relative}${P_ERROR}'.${P_END}" + echo -e "${P_ERROR}${result}${P_END}" + else + echo -e "${P_OK}Copied '${P_END}${source_file_relative}${P_OK}' to '${P_END}${target_file_relative}${P_OK}'.${P_END}" + fi + + #split on commas + IFS_OLD="$IFS" + IFS=, to_formats=($converttoformats) + IFS="$IFS_OLD" + for target_format in ${to_formats[@]}; do + if [ -z "$target_format" ]; then + # skip blanks + continue; + fi + + # convert to target format + # ------------------------ + newpath="${source_file/$OSCALDIR\/src\//}" # strip off src + newpath="${newpath/\/$source_format\///$target_format/}" # change path from old to new format dir + newpath="${newpath%.*}" # strip extension + + case $target_format in + xml) + target_file="$WORKING_DIR/${newpath}-min.${target_format}" + ;; + json) + target_file="$WORKING_DIR/${newpath}-min.${target_format}" + ;; + *) + echo -e "${P_WARN}Conversion from '${source_format} to '${target_format^^}' is unsupported for '${P_END}${source_file_relative}${P_OK}'.${P_END}" + continue; + esac + + target_dir=${target_file%/*} # remove filename + mkdir -p "$target_dir" + + target_file_relative=$(get_rel_path "${WORKING_DIR}" "$target_file") + if [ "$VERBOSE" = "true" ]; then + echo -e "${P_INFO}Performing ${source_format^^}->${target_format^^} conversion of '${P_END}${source_file_relative}${P_INFO}' to '${P_END}${target_file_relative}${P_INFO}'.${P_END}" + fi + + result=$(convert_to_format_and_validate "$source_file" "$target_file" "$source_format" "$target_format" "$model") + if [ -n "$result" ]; then + echo -e "${result}" + fi + cmd_exitcode=$? + if [ $cmd_exitcode != 0 ]; then + exitcode=1 + continue; + else + echo -e "${P_OK}Converted ${source_format^^} '${P_END}${source_file_relative}${P_OK}' to ${target_format^^} as '${P_END}${target_file_relative}${P_OK}'.${P_END}" + fi + + # Format specific post-processing + case $target_format in + json) + # Remove extra slashes + # perl -pi -e 's,\\/,/,g' "${dest}" + # translate OSCAL mime types + perl -pi -e 's,(application/oscal\.[a-z]+\+)xml\",\1json\",g' "${target_file}" + # relative content paths + # translate path names, starting first with the xml directory, then the filename + perl -pi -e 's,(\.\./[^\"]+(?=/xml/))/xml/,\1/json/,g' "${target_file}" + perl -pi -e 's,(\.\./[^\"]+(?=/json/)[^\"]+(?=.xml\")).xml\",\1.json\",g' "${target_file}" + perl -pi -e 's,(\"[^/\"]+(?=\.xml\")).xml\",\1.json\",g' "${target_file}" + + # produce pretty JSON + target_file_pretty="$WORKING_DIR/${newpath}.${target_format}" + target_file_pretty_relative=$(get_rel_path "${WORKING_DIR}" "$target_file_pretty") + result=$(jq . "$target_file" > "$target_file_pretty" 2>&1) + if [ $? -ne 0 ]; then echo -e "${P_ERROR}${result}${P_END}" - else - echo -e "${P_OK}Copied '${P_END}${file_relative}${P_OK}' to '${P_END}${dest_relative}${P_OK}'.${P_END}" + echo -e "${P_ERROR}Unable to execute jq on '${P_END}${target_file_pretty_relative}${P_ERROR}'.${P_END}" + exitcode=1 + continue fi - IFS="," - for altformat in "$converttoformats"; do - newpath="${file/$OSCALDIR\/src\//}" # strip off src - newpath="${newpath/\/$format\///$altformat/}" # change path from old to new format dir - newpath="${newpath%.*}" # strip extension - - converter="$WORKING_DIR/$altformat/convert/oscal_${model}_${format}-to-${altformat}-converter.xsl" - converter_relative=$(realpath --relative-to="${WORKING_DIR}" "$converter") - - - case $format in - xml) - dest="$WORKING_DIR/${newpath}-min.${altformat}" - dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest") - if [ "$VERBOSE" = "true" ]; then - echo -e "${P_INFO}Generating ${altformat^^} file '${P_END}${dest_relative}${P_INFO}' from '${P_END}${file_relative}${P_INFO}' using converter '${P_END}${converter_relative}${P_INFO}'.${P_END}" - fi - result=$(xsl_transform "$converter" "$file" "$dest" 2>&1) - ;; - json) - dest="$WORKING_DIR/${newpath}.${altformat}" - dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest") - if [ "$VERBOSE" = "true" ]; then - echo -e "${P_INFO}Generating ${altformat^^} file '${P_END}${dest_relative}${P_INFO}' from '${P_END}${file_relative}${P_INFO}' using converter '${P_END}${converter_relative}${P_INFO}'.${P_END}" - fi - result=$(xsl_transform "$converter" "" "$dest" "-it" "json-file=${file}" 2>&1) - ;; - *) - echo -e "${P_WARN}Conversion from '${format} to '${altformat^^}' is unsupported for '${P_END}${file_relative}${P_OK}'.${P_END}" - continue; - esac - cmd_exitcode=$? - if [ $cmd_exitcode -ne 0 ]; then - echo -e "${P_ERROR}Content conversion to ${altformat^^} failed for '${P_END}${file_relative}${P_ERROR}' using converter '${P_END}${converter_relative}${P_ERROR}'.${P_END}" - echo -e "${P_ERROR}${result}${P_END}" - exitcode=1 - continue - else - if [ "$VERBOSE" = "true" ]; then - echo -e "${P_OK}Content conversion to ${altformat^^} succeeded for '${P_END}${file_relative}${P_OK}'.${P_END}" - fi - fi - - # Format specific post-processing - case $altformat in - json) - # Remove extra slashes - # perl -pi -e 's,\\/,/,g' "${dest}" - # translate OSCAL mime types - perl -pi -e 's,(application/oscal\.[a-z]+\+)xml\",\1json\",g' "${dest}" - # relative content paths - # translate path names, starting first with the xml directory, then the filename - perl -pi -e 's,(\.\./[^\"]+(?=/xml/))/xml/,\1/json/,g' "${dest}" - perl -pi -e 's,(\.\./[^\"]+(?=/json/)[^\"]+(?=.xml\")).xml\",\1.json\",g' "${dest}" - perl -pi -e 's,(\"[^/\"]+(?=\.xml\")).xml\",\1.json\",g' "${dest}" - -# cp "${dest}.tmp" "${dest}" - - # validate generated file - schema="$WORKING_DIR/json/schema/oscal_${model}_schema.json" - schema_relative=$(realpath --relative-to="${WORKING_DIR}" "$schema") - result=$(validate_json "$schema" "$dest") - cmd_exitcode=$? - if [ $cmd_exitcode -ne 0 ]; then - echo -e "${P_ERROR}JSON Schema validation failed for '${P_END}${dest_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}" - echo -e "${P_ERROR}${result}${P_END}" - exitcode=1 - continue - else - echo -e "${P_OK}JSON Schema validation passed for '${P_END}${dest_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}" - fi - - # produce pretty JSON - dest_pretty="$WORKING_DIR/${newpath}.${altformat}" - dest_pretty_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest_pretty") - result=$(jq . "$dest" > "$dest_pretty" 2>&1) - if [ $? -ne 0 ]; then - echo -e "${P_ERROR}Unable to execute jq on '${P_END}${dest_pretty_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}" - echo -e "${P_ERROR}${result}${P_END}" - exitcode=1 - continue - fi - # remove carriage returns - perl -pi -e 's,\r,,g' "$dest_pretty" - - result=$(validate_json "$schema" "$dest_pretty" 2>&1) - cmd_exitcode=$? - if [ $cmd_exitcode -ne 0 ]; then - echo -e "${P_ERROR}JSON Schema validation failed for '${P_END}${dest_pretty_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}" - echo -e "${P_ERROR}${result}${P_END}" - exitcode=1 - continue - else - echo -e "${P_OK}JSON Schema validation passed for '${P_END}${dest_pretty_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}" - fi - - # produce yaml - newpath="${newpath/\/json\///yaml/}" # change path - dest_pretty="$WORKING_DIR/${newpath}.yaml" - dest_pretty_dir=${dest_pretty%/*} # remove filename - mkdir -p "$dest_pretty_dir" - prettyjson --nocolor=1 --indent=2 --inline-arrays=1 "$dest" > "$dest_pretty" - ;; - xml) - # validate generated file - schema="$WORKING_DIR/xml/schema/oscal_${model}_schema.xsd" - schema_relative=$(realpath --relative-to="${WORKING_DIR}" "$schema") - result=$(xmllint --noout --schema "$schema" "$dest" 2>&1) - cmd_exitcode=$? - if [ $cmd_exitcode -ne 0 ]; then - echo -e "${P_ERROR}XML Schema validation failed for '${P_END}${dest_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}" - echo -e "${P_ERROR}${result}${P_END}" - exitcode=1 - else - echo -e "${P_OK}XML Schema validation passed for '${P_END}${dest_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}" - fi - ;; - *) - echo -e "${P_WARN}Post processing of '${altformat^^}' is unsupported for '${P_END}${dest_relative}${P_OK}'.${P_END}" - continue; - esac - done - done - fi -done < "$OSCALDIR/build/ci-cd/config/content" + # remove carriage returns + perl -pi -e 's,\r,,g' "$target_file_pretty" + + result=$(validate_content "$target_file_pretty" "json" "$model") + if [ $? -ne 0 ]; then + echo -e "${P_ERROR}${result}${P_END}" + echo -e "${P_ERROR}Unable to execute jq on '${P_END}${target_file_pretty_relative}${P_ERROR}'.${P_END}" + exitcode=1 + continue + fi -shopt -u nullglob -shopt -u globstar + # produce yaml + newpath="${newpath/\/json\///yaml/}" # change path + target_file_yaml="$WORKING_DIR/${newpath}.yaml" + target_file_yaml_dir=${target_file_yaml%/*} # remove filename + mkdir -p "$target_file_yaml_dir" + prettyjson --nocolor=1 --indent=2 --inline-arrays=1 "$target_file" > "$target_file_yaml" + ;; + xml) + # do nothing + ;; + *) + echo -e "${P_WARN}Post processing of '${altformat^^}' is unsupported for '${P_END}${dest_relative}${P_OK}'.${P_END}" + continue; + esac + done +done exit $exitcode diff --git a/build/ci-cd/generate-content-converters.sh b/build/ci-cd/generate-content-converters.sh index bd34b37873..2c188b4898 100755 --- a/build/ci-cd/generate-content-converters.sh +++ b/build/ci-cd/generate-content-converters.sh @@ -1,15 +1,8 @@ #!/bin/bash -if [[ -z "$OSCALDIR" ]]; then - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - source "$DIR/include/common-environment.sh" +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh" fi -source "$OSCALDIR/build/ci-cd/include/saxon-init.sh" - -# Option defaults -WORKING_DIR="${OSCALDIR}" -VERBOSE=false -HELP=false usage() { # Function: Print a help message. cat << EOF @@ -70,6 +63,7 @@ if [ "$#" -ne 0 ]; then formats[$i]="xml,json" done else + IFS_OLD="$IFS" while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; do [[ "$path" =~ ^[[:space:]]*# ]] && continue # remove leading space @@ -79,19 +73,18 @@ else path_absolute="$OSCALDIR"/"$path" - IFS_OLD=$IFS IFS= # disable word splitting for metaschema in $path_absolute do - paths+=("$metaschema") - formats+=("$gen_converter") + paths+=($metaschema) + formats+=($gen_converter) done - IFS=$IFS_OLD done < "$OSCALDIR/build/ci-cd/config/metaschema" + IFS="$IFS_OLD" fi exitcode=0 -for i in "${!paths[@]}"; do +for i in ${!paths[@]}; do metaschema="${paths[$i]}" gen_converter="${formats[$i]}" @@ -99,19 +92,20 @@ for i in "${!paths[@]}"; do extension="${filename##*.}" filename="${filename%.*}" model="${filename/_metaschema/}" - metaschema_relative=$(realpath --relative-to="${OSCALDIR}" "$metaschema") + metaschema_relative=$(get_rel_path "${OSCALDIR}" "$metaschema") #split on commas - IFS_OLD=$IFS - IFS=, #read -a formats <<< "$gen_schema" - for target_format in ${gen_converter}; do - if [ -z "$target_format" ]; then + IFS_OLD="$IFS" + IFS=, gen_formats=($gen_converter) + IFS="$IFS_OLD" + for format in ${gen_formats[@]}; do + if [ -z "$format" ]; then # skip blanks continue; fi - # Run the XSL template for the format - case $target_format in + # setup source and target formats + case $format in xml) source_format="json" ;; @@ -119,29 +113,45 @@ for i in "${!paths[@]}"; do source_format="xml" ;; *) - echo -e "${P_WARN}Generating converter from '${source_format^^}' to '${target_format^^}' is unsupported for '${P_END}${metaschema_relative}${P_WARN}'.${P_END}" + >&2 echo -e "${P_ERROR}Generating converter from '${P_END}${source_format^^}${P_ERROR}' to '${P_END}${target_format^^}${P_ERROR}' is unsupported for '${P_END}${metaschema_relative}${P_ERROR}'.${P_END}" continue; ;; esac + target_format="$format" converter="$WORKING_DIR/${target_format}/convert/${model}_${source_format}-to-${target_format}-converter.xsl" - converter_relative=$(realpath --relative-to="${WORKING_DIR}" "$converter") + # ensure the converter directory exists + mkdir -p "$(dirname "$converter")" + converter_relative=$(get_rel_path "${WORKING_DIR}" "$converter") - if [ "$VERBOSE" = "true" ]; then - echo -e "${P_INFO}Generating ${source_format^^} to ${target_format^^} converter for '${P_END}${metaschema_relative}${P_INFO}' as '${P_END}${converter_relative}${P_INFO}'.${P_END}" - fi - result=$(xsl_transform "$OSCALDIR/build/metaschema/$source_format/produce-${source_format}-converter.xsl" "$metaschema" "$converter" 2>&1) +# echo "source: ${source_format}" +# echo "target: ${target_format}" +# echo "metaschema: ${metaschema}" +# echo "converter: ${converter}" + args=() + args+=("--source-format" "${source_format}") + args+=("--target-format" "${target_format}") + + if [ "$VERBOSE" == "true" ]; then + args+=("-v") + fi + + args+=("$metaschema") + args+=("$converter") + + result=$("$OSCALDIR/build/metaschema/scripts/generate-content-converter.sh" "${args[@]}" 2>&1) cmd_exitcode=$? if [ $cmd_exitcode -ne 0 ]; then - echo -e "${P_ERROR}Generating ${source_format^^} to ${target_format^^} converter failed for '${P_END}${metaschema_relative}${P_ERROR}'.${P_END}" echo -e "${P_ERROR}${result}${P_END}" + echo -e "${P_ERROR}Generation of ${source_format^^} to ${target_format^^} converter failed for '${P_END}${metaschema_relative}${P_ERROR}'.${P_END}" exitcode=1 else - echo -e "${P_OK}Generating ${source_format^^} to ${target_format^^} converter passed for '${P_END}${metaschema_relative}${P_OK}'.${P_END}" + echo -e "${result}" fi done IFS=$IFS_OLD done exit $exitcode + diff --git a/build/ci-cd/generate-model-documentation.sh b/build/ci-cd/generate-model-documentation.sh index 15761d21a1..f2cdea572c 100755 --- a/build/ci-cd/generate-model-documentation.sh +++ b/build/ci-cd/generate-model-documentation.sh @@ -1,15 +1,12 @@ #!/bin/bash -if [[ -z "$OSCALDIR" ]]; then - DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" - source "$DIR/include/common-environment.sh" +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh" fi -source "$OSCALDIR/build/ci-cd/include/saxon-init.sh" + +source "$OSCALDIR/build/metaschema/scripts/include/init-saxon.sh" # Option defaults -WORKING_DIR="${OSCALDIR}" -VERBOSE=false -HELP=false usage() { # Function: Print a help message. cat </dev/null && pwd)/include/init-oscal.sh" fi -#source "$OSCALDIR/build/metaschema/scripts/include/saxon-init.sh" -source "$OSCALDIR/build/metaschema/scripts/include/init-validate-json.sh" - usage() { # Function: Print a help message. cat << EOF Usage: $0 [options] [metaschema paths] @@ -98,7 +95,6 @@ for i in ${!paths[@]}; do metaschema_relative=$(get_rel_path "${OSCALDIR}" "${metaschema}") #split on commas - IFS_OLD="$IFS" IFS=, gen_formats=($gen_schema) IFS="$IFS_OLD" @@ -127,7 +123,7 @@ for i in ${!paths[@]}; do mkdir -p "$(dirname "$schema")" schema_relative=$(get_rel_path "${WORKING_DIR}" "${schema}") - if [ "$VERBOSE" = "true" ]; then + if [ "$VERBOSE" == "true" ]; then echo -e "${P_INFO}Generating ${format^^} schema for '${P_END}${metaschema_relative}${P_INFO}' as '${P_END}${schema_relative}${P_INFO}'.${P_END}" fi @@ -141,14 +137,15 @@ for i in ${!paths[@]}; do args+=("-v") fi - result=$(IFS=$' ' "$OSCALDIR/build/metaschema/scripts/generate-schema.sh" "${args[@]}" 2>&1) + result=$("$OSCALDIR/build/metaschema/scripts/generate-schema.sh" "${args[@]}" 2>&1) cmd_exitcode=$? if [ $cmd_exitcode -ne 0 ]; then echo -e "${P_ERROR}Generation of ${format^^} schema failed for '${P_END}${metaschema_relative}${P_ERROR}'.${P_END}" echo -e "${P_ERROR}${result}${P_END}" exitcode=1 else - if [ "$VERBOSE" = "true" ]; then + echo -e "${result}" + if [ "$VERBOSE" == "true" ]; then echo -e "${P_OK}Generation of ${format^^} schema passed for '${P_END}${metaschema_relative}${P_OK}'.${P_END}" fi fi diff --git a/build/ci-cd/include/convert-and-validate-content.sh b/build/ci-cd/include/convert-and-validate-content.sh new file mode 100644 index 0000000000..17888ba641 --- /dev/null +++ b/build/ci-cd/include/convert-and-validate-content.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +validate_content() { + local target_file="$1"; shift + local target_format="$1"; shift + local model="$1"; shift + + # validate target + case $target_format in + xml) + target_schema="$WORKING_DIR/$target_format/schema/oscal_${model}_schema.xsd" + result=$(validate_xml "$target_schema" "$target_file") + ;; + json) + target_schema="$WORKING_DIR/$target_format/schema/oscal_${model}_schema.json" + result=$(validate_json "$target_schema" "$target_file") + ;; + *) + echo -e "${P_WARN}Unsupported schema format '${target_format^^}'.${P_END}" + return 4; + ;; + esac + + cmd_exitcode=$? + if [ $cmd_exitcode -ne 0 ]; then + echo -e "${P_ERROR}${result}${P_END}" + echo -e "${P_ERROR}${target_format^^} Schema validation failed for '${P_END}${target_file}${P_ERROR}' using schema '${P_END}${target_schema}${P_ERROR}'.${P_END}" + return 2; + else + if [ "$VERBOSE" = "true" ]; then + echo -e "${P_OK}${target_format^^} Schema validation passed for '${P_END}${target_file}${P_OK}'.${P_END}" + fi + fi + return 0; +} + +convert_to_format_and_validate() { + local source_file="$1"; shift + local target_file="$1"; shift + local source_format="$1"; shift + local target_format="$1"; shift + local model="$1"; shift + + # get the schema to use for validating the target + case $target_format in + xml) + target_converter="$WORKING_DIR/xml/convert/oscal_${model}_json-to-xml-converter.xsl" + ;; + json) + target_converter="$WORKING_DIR/json/convert/oscal_${model}_xml-to-json-converter.xsl" + ;; + *) + echo -e "${P_WARN}Unsupported target format '${target_format^^}'.${P_END}" + return 3; + ;; + esac + + # convert source to target format +# echo "Source: $source_file" +# echo "Target: $target_file" +# echo "Source Format: $source_format" +# echo "Target Format: $target_format" +# echo "Converter: $target_converter" + + result=$(convert_content "$source_file" "$target_file" "$source_format" "$target_format" "$target_converter") + cmd_exitcode=$? + if [ $cmd_exitcode != 0 ]; then + echo -e "${P_ERROR}${result}${P_END}" + echo -e "${P_ERROR}${source_format^^}->${target_format^^} conversion failed for '${P_END}${source_file}${P_ERROR}'.${P_END}" + return 1; + fi + + result=$(validate_content "$target_file" "$target_format" "$model") + echo -ne "${result}" + cmd_exitcode=$? + if [ $cmd_exitcode != 0 ]; then + return 1; + fi + return 0; +} + diff --git a/build/ci-cd/include/init-oscal.sh b/build/ci-cd/include/init-oscal.sh index 7943c38bdb..def131ecbe 100755 --- a/build/ci-cd/include/init-oscal.sh +++ b/build/ci-cd/include/init-oscal.sh @@ -1,13 +1,14 @@ #!/bin/bash # Setup script environment -if [ -z ${METASCHEMA_SCRIPT_INIT+x} ]; then +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then # Get location of this script and set the OSCAL directory as a relative path - DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" - OSCALDIR=$(cd "$DIR/../../.."; pwd) + OSCALDIR="$(cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/../../.."; pwd)" WORKING_DIR="${OSCALDIR}" source "$OSCALDIR/build/metaschema/scripts/include/common-environment.sh" + + OSCAL_SCRIPT_INIT=true fi diff --git a/build/ci-cd/package-release.sh b/build/ci-cd/package-release.sh index aa852cff2f..c2b20454e9 100755 --- a/build/ci-cd/package-release.sh +++ b/build/ci-cd/package-release.sh @@ -1,8 +1,7 @@ #!/bin/bash -if [[ -z "$OSCALDIR" ]]; then - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - source "$DIR/include/common-environment.sh" +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh" fi if [ -z "$1" ]; then diff --git a/build/ci-cd/run-all.sh b/build/ci-cd/run-all.sh index 331d45e7e8..068f2da0ec 100755 --- a/build/ci-cd/run-all.sh +++ b/build/ci-cd/run-all.sh @@ -1,8 +1,7 @@ #!/bin/bash # determines the OSCAL directory path -if [[ -z "$OSCALDIR" ]]; then - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - source "$DIR/include/common-environment.sh" +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh" fi # Option defaults @@ -13,8 +12,6 @@ PERFORM_CONTENT_CONVERSION=YES RUN_UNITTESTS=YES KEEP_TEMP_SCRATCH_DIR=false WORKING_DIR="${OSCALDIR}" -VERBOSE=false -HELP=false usage() { # Function: Print a help message. cat << EOF diff --git a/build/ci-cd/run-unittests.sh b/build/ci-cd/run-unittests.sh index 9e3d7efc1b..f03dbbf6c3 100755 --- a/build/ci-cd/run-unittests.sh +++ b/build/ci-cd/run-unittests.sh @@ -1,16 +1,13 @@ #!/bin/bash # Setup OSCAL environment -if [[ -z "$OSCALDIR" ]]; then - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - source "$DIR/include/common-environment.sh" + +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh" fi -source "$OSCALDIR/build/ci-cd/include/init-validate-json.sh" +source "$OSCALDIR/build/metaschema/scripts/include/init-validate-content.sh" # Option defaults -WORKING_DIR="${OSCALDIR}" -VERBOSE=false -HELP=false usage() { # Function: Print a help message. cat << EOF diff --git a/build/ci-cd/validate-content-conversion-round-trips.sh b/build/ci-cd/validate-content-conversion-round-trips.sh index 3846bf956b..c1faf420f3 100755 --- a/build/ci-cd/validate-content-conversion-round-trips.sh +++ b/build/ci-cd/validate-content-conversion-round-trips.sh @@ -1,21 +1,17 @@ #!/bin/bash -# set the OSCAL directory and pass in common environment -if [[ -z "$OSCALDIR" ]]; then - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - source "$DIR/include/common-environment.sh" + +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh" fi -# initialize Saxon and get the path -source "$OSCALDIR/build/ci-cd/include/saxon-init.sh" +source "$OSCALDIR/build/metaschema/scripts/include/init-validate-content.sh" +source "$OSCALDIR/build/ci-cd/include/convert-and-validate-content.sh" # Catalog round trip from XML -> JSON -> XML MSYS_NO_PATHCONV=1 # Option defaults KEEP_TEMP_SCRATCH_DIR=false -WORKING_DIR="${OSCALDIR}" -VERBOSE=false -HELP=false usage() { # Function: Print a help message. cat << EOF @@ -103,122 +99,132 @@ fi # XML->JSON Roundtrip conversions tests ################################################################################################################### -# default to test passed at zero -exitcode=0 -shopt -s nullglob -shopt -s globstar -while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do +IFS_OLD="$IFS" +while IFS="|" read path_glob format model converttoformats || [[ -n "$path_glob" ]]; do shopt -s extglob - [[ "$path" =~ ^[[:space:]]*# ]] && continue + [[ "$path_glob" =~ ^[[:space:]]*# ]] && continue # remove leading space - path="${path##+([[:space:]])}" + path_glo="${path_glob##+([[:space:]])}" # remove trailing space converttoformats="${converttoformats%%+([[:space:]])}" shopt -u extglob - if [[ ! -z "$path" ]]; then - files_to_process="$OSCALDIR/$path" - - IFS= # disable word splitting - #loop through the files - for file in $files_to_process - do - # get the base file name - file_basename=$(basename $file) - file_relative=$(realpath --relative-to="$OSCALDIR" "$file") - - # debuggging statements, shows what is processing - #printf 'path: %s\n' "$file" - #printf 'file name: %s\n' "$file_basename" - #printf 'format: %s\n' "$format" - #printf 'model: %s\n' "$model" - #printf 'convert-to: %s\n' "$converttoformats" - - if [ "$format" == "xml" ]; then - # XML -> JSON -> XML round trip testing - # transformation from source XML to target JSON - converter="$WORKING_DIR/json/convert/oscal_${model}_xml-to-json-converter.xsl" - - to_json_file="${SCRATCH_DIR}/roundtrip/${file_basename}-to.json" - result=$(xsl_transform "$converter" "$file" "$to_json_file" 2>&1) - - # check the exit code for the conversion - cmd_exitcode=$? - if [ $cmd_exitcode != 0 ]; then - echo -e "${P_ERROR}XML->JSON conversion failed for '${P_END}${file_relative}${P_ERROR}'.${P_END}" - echo -e "${P_ERROR}${result}${P_END}" - exitcode=1 - continue; - else - if [ "$VERBOSE" = "true" ]; then - echo -e "${P_OK}Converted XML '${P_END}${file_relative}${P_OK}' to JSON '${P_END}${to_json_file}${P_OK}'.${P_END}" - fi - fi - - back_to_xml_file="${SCRATCH_DIR}/roundtrip/${file_basename}-to-json-and-back-to.xml" - - # transformation of JSON back to XML - converter="$WORKING_DIR/xml/convert/oscal_${model}_json-to-xml-converter.xsl" - converter_path="$(realpath "$converter")" - back_to_xml_file_relative="$(realpath --relative-to="$PWD" "$back_to_xml_file")" - - # Make the json file relative to the converter - converter_dir="$(dirname "$converter")" - json_file_path="$(realpath "$to_json_file")" - - result=$(xsl_transform "$converter_path" "" "$back_to_xml_file" "-it" "json-file=${json_file_path}" 2>&1) - - # check the exit code for the conversion - cmd_exitcode=$? - if [ $cmd_exitcode != 0 ]; then - echo -e "${P_ERROR}JSON->XML conversion failed for '${P_END}${to_json_file}${P_ERROR}'.${P_END}" - echo -e "${P_ERROR}${result}${P_END}" - exitcode=1 - continue; - else - if [ "$VERBOSE" = "true" ]; then - echo -e "${P_OK}Converted JSON '${P_END}${to_json_file}${P_OK}' to XML for '${P_END}${back_to_xml_file}${P_OK}'.${P_END}" - fi - fi + [ -z "$path_glob" ] && continue; + + path_absolute="$OSCALDIR"/"$path_glob" + + for path in $path_absolute; do +# echo "Path: $path" +# echo "Format: $format" +# echo "Model: $model" +# echo "Convert to: $converttoformats" + + paths+=("$path") + formats+=("$format") + models+=("$model") + conversion_formats+=("$converttoformats") + done +done < "${OSCALDIR}/build/ci-cd/config/content" +IFS="$IFS_OLD" + +#echo "Paths: ${paths[@]}" +#echo "Formats: ${formats[@]}" +#echo "Models: ${models[@]}" +#echo "Convert To: ${conversion_formats[@]}" - # Validate the resulting XML - schema="$WORKING_DIR/xml/schema/oscal_${model}_schema.xsd" - schema_relative="$(realpath --relative-to="${WORKING_DIR}" "$schema")" - result="$(xmllint --noout --schema "$schema" "$back_to_xml_file" 2>&1)" - cmd_exitcode=$? - if [ $cmd_exitcode -ne 0 ]; then - echo -e "${P_ERROR}XML Schema validation failed for '${P_END}${back_to_xml_file}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}" - echo -e "${P_ERROR}${result}${P_END}" - exitcode=1 - continue; - else - if [ "$VERBOSE" = "true" ]; then - echo -e "${P_OK}XML Schema validation passed for '${P_END}${back_to_xml_file}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}" - fi - fi - - # compare the XML files to see if there is data loss - if [ "$VERBOSE" = "true" ]; then - echo -e "${P_INFO}Checking XML->JSON->XML conversion for '${P_END}${file_relative}${P_INFO}'.${P_END}" - fi +shopt -s nullglob +shopt -s globstar - result=$(python ${OSCALDIR}/build/ci-cd/python/xmlComparison.py "$back_to_xml_file" "$file" 2>&1) - cmd_exitcode=$? - if [ $cmd_exitcode != 0 ]; then - echo -e "${P_ERROR}XML round-trip comparison failed for '${P_END}${file_relative}${P_ERROR}' against '${P_END}${back_to_xml_file}${P_ERROR}'.${P_END}" - echo -e "${P_ERROR}${result}${P_END}" - if [ "$VERBOSE" != "true" ]; then - echo -e " ${P_ERROR}Used interim JSON file '${P_END}${to_json_file}${P_ERROR}'.${P_END}" - fi - exitcode=1 - else - echo -e "${P_OK}XML round-trip comparison succeeded for '${P_END}${file_relative}${P_OK}' against '${P_END}${back_to_xml_file}${P_OK}'.${P_END}" +exitcode=0 +for i in ${!paths[@]}; do +# echo "Index: $i" + source_file="${paths[$i]}" + source_format="${formats[$i]}" + model="${models[$i]}" + converttoformats="${conversion_formats[$i]}" + + # get the base file name + source_file_basename=$(basename $source_file) + source_file_relative=$(get_rel_path "$OSCALDIR" "$source_file") + + # debuggging statements, shows what is processing +# printf 'path: %s\n' "$file" +# printf 'file name: %s\n' "$file_basename" +# printf 'Source format: %s\n' "$source_format" +# printf 'model: %s\n' "$model" +# printf 'convert-to: %s\n' "$converttoformats" + + source_schema="$WORKING_DIR/$source_format/schema/oscal_${model}_schema.xsd" + + #split on commas + IFS_OLD="$IFS" + IFS=, to_formats=($converttoformats) + IFS="$IFS_OLD" + for target_format in ${to_formats[@]}; do + if [ -z "$target_format" ]; then + # skip blanks + continue; + fi + + # convert to target format + target_file="${SCRATCH_DIR}/roundtrip/${source_file_basename}-to.${target_format}" + result=$(convert_to_format_and_validate "$source_file" "$target_file" "$source_format" "$target_format" "$model") + cmd_exitcode=$? + if [ -n "$result" ]; then + echo -e "${result}" + fi + + if [ $cmd_exitcode != 0 ]; then + exitcode=1 + continue; + else + echo -e "${P_OK}Converted ${source_format^^} '${P_END}${source_file_relative}${P_OK}' to ${target_format^^} as '${P_END}${target_file}${P_OK}'.${P_END}" + fi + + # convert back to source format + roundtrip_file="${SCRATCH_DIR}/roundtrip/${source_file_basename}-to-${target_format}-back-to.${source_format}" + result=$(convert_to_format_and_validate "$target_file" "$roundtrip_file" "$target_format" "$source_format" "$model") + cmd_exitcode=$? + if [ -n "$result" ]; then + echo -e "${result}" + fi + + if [ $cmd_exitcode != 0 ]; then + exitcode=1 + continue; + else + echo -e "${P_OK}Converted ${source_format^^} '${P_END}${target_file}${P_OK}' to ${target_format^^} as '${P_END}${roundtrip_file}${P_OK}'.${P_END}" + fi + + # compare the XML files to see if there is data loss + if [ "$VERBOSE" = "true" ]; then + echo -e "${P_INFO}Checking ${source_format^^}->${target_format^^}->${source_format^^} conversion for '${P_END}${source_file_relative}${P_INFO}'.${P_END}" + fi + + case $source_format in + xml) + result=$(python ${OSCALDIR}/build/ci-cd/python/xmlComparison.py "$roundtrip_file" "$source_file" 2>&1) + ;; + json) + result=$(json-diff "$source_file" "$roundtrip_file" 2>&1) + ;; + *) + echo -e "${P_WARN}Unsupported source compairison format '${source_format^^}'.${P_END}" + return 4; + ;; + esac + cmd_exitcode=$? + if [ $cmd_exitcode != 0 ]; then + echo -e "${P_ERROR}${result}${P_END}" + echo -e "${P_ERROR}${source_format^^}->${target_format^^}->${source_format^^} round-trip comparison failed for '${P_END}${source_file_relative}${P_ERROR}' against '${P_END}${roundtrip_file}${P_ERROR}'.${P_END}" + if [ "$VERBOSE" != "true" ]; then + echo -e " ${P_ERROR}Using interim file '${P_END}${target_file}${P_ERROR}'.${P_END}" fi - fi - done - fi -done < "$OSCALDIR/build/ci-cd/config/content" #inserts the config of files to parse -shopt -u nullglob -shopt -u globstar + exitcode=1 + else + echo -e "${P_OK}${source_format^^}->${target_format^^}->${source_format^^} round-trip comparison succeeded for '${P_END}${source_file_relative}${P_OK}' against '${P_END}${roundtrip_file}${P_OK}'.${P_END}" + fi + done +done exit $exitcode diff --git a/build/ci-cd/validate-content.sh b/build/ci-cd/validate-content.sh index 87437e41b8..4588f24f68 100755 --- a/build/ci-cd/validate-content.sh +++ b/build/ci-cd/validate-content.sh @@ -1,11 +1,11 @@ #!/bin/bash # Setup OSCAL environment -if [[ -z "$OSCALDIR" ]]; then - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - source "$DIR/include/common-environment.sh" + +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh" fi -source "$OSCALDIR/build/ci-cd/include/init-validate-json.sh" +source "$OSCALDIR/build/metaschema/scripts/include/init-validate-content.sh" # Option defaults WORKING_DIR="${OSCALDIR}" diff --git a/build/ci-cd/validate-metaschema.sh b/build/ci-cd/validate-metaschema.sh index b646bb789b..cf114e4173 100755 --- a/build/ci-cd/validate-metaschema.sh +++ b/build/ci-cd/validate-metaschema.sh @@ -1,10 +1,10 @@ #!/bin/bash -if [[ -z "$OSCALDIR" ]]; then - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - source "$DIR/include/common-environment.sh" + +if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh" fi -source "$OSCALDIR/build/ci-cd/include/schematron-init.sh" +source "$OSCALDIR/build/metaschema/scripts/include/init-schematron.sh" # Option defaults KEEP_TEMP_SCRATCH_DIR=false @@ -96,7 +96,7 @@ if [ "$VERBOSE" = "true" ]; then fi # compile the schematron -metaschema_lib="$OSCALDIR/build/metaschema/lib" +metaschema_lib="$OSCALDIR/build/metaschema/toolchains/oscal-m2/lib" schematron="$metaschema_lib/metaschema-check.sch" compiled_schematron="${SCRATCH_DIR}/metaschema-schematron-compiled.xsl"