Completed first pass through CI/CD scripts in support of usnistgov#596

build/ci-cd/config/content

@@ -1,6 +1,7 @@
 # path to source|format of source|model of source|format(s) to convert to
 src/content/ssp-example/ssp-example.json|json|ssp|xml
-src/content/fedramp.gov/xml/*catalog.xml|xml|catalog|json
+src/content/fedramp.gov/xml/FedRAMP_catalog.xml|xml|catalog|json
 src/content/fedramp.gov/xml/*profile.xml|xml|profile|json
 src/content/nist.gov/SP800-53/rev4/xml/*catalog.xml|xml|catalog|json
 src/content/nist.gov/SP800-53/rev4/xml/*profile.xml|xml|profile|json
+

build/ci-cd/config/metaschema

@@ -3,3 +3,4 @@ src/metaschema/oscal_catalog_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_profile_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_component_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_ssp_metaschema.xml|xml,json|xml,json|xml,json
+

build/ci-cd/copy-and-convert-content.sh

@@ -1,16 +1,14 @@
 #!/bin/bash
 
-if [[ -z "$OSCALDIR" ]]; then
-    DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-    source "$DIR/include/common-environment.sh"
+
+if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then
+    source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh"
 fi
-source "$OSCALDIR/build/ci-cd/include/saxon-init.sh"
-source "$OSCALDIR/build/ci-cd/include/init-validate-json.sh"
+
+source "$OSCALDIR/build/metaschema/scripts/include/init-validate-content.sh"
+source "$OSCALDIR/build/ci-cd/include/convert-and-validate-content.sh"
 
 # Option defaults
-WORKING_DIR="${OSCALDIR}"
-VERBOSE=false
-HELP=false
 
 usage() {                                      # Function: Print a help message.
   cat << EOF
@@ -19,8 +17,6 @@ Usage: $0 [options]
 -h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
 -v                                Provide verbose output
---keep-temp-scratch-dir           If a scratch directory is automatically
-                                  created, it will not be automatically removed.
 EOF
 }
 
@@ -72,164 +68,177 @@ if [ $? -ne 0 ]; then
   exit 1
 fi
 
-exitcode=0
-shopt -s nullglob
-shopt -s globstar
-while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
+IFS_OLD="$IFS"
+while IFS="|" read path_glob format model converttoformats || [[ -n "$path_glob" ]]; do
   shopt -s extglob
-  [[ "$path" =~ ^[[:space:]]*# ]] && continue
+  [[ "$path_glob" =~ ^[[:space:]]*# ]] && continue
   # remove leading space
-  path="${path##+([[:space:]])}"
+  path_glo="${path_glob##+([[:space:]])}"
   # remove trailing space
   converttoformats="${converttoformats%%+([[:space:]])}"
   shopt -u extglob
 
-  if [[ ! -z "$path" ]]; then
-    files_to_process="$OSCALDIR"/"$path"
-    IFS= # disable word splitting
-    for file in $files_to_process
-    do
-      file_relative=$(realpath --relative-to="${OSCALDIR}" "$file")
-      dest="$WORKING_DIR/${file/$OSCALDIR\/src\//}"
-      dest_dir=${dest%/*} # remove filename
-      mkdir -p "$dest_dir"
-      dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest")
-      result=$(cp "$file" "$dest" 2>&1)
-      cmd_exitcode=$?
-      if [ $cmd_exitcode -ne 0 ]; then
-        echo -e "${P_ERROR}Unable to copy '${P_END}${file_relative}${P_ERROR}' to '${P_END}${dest_relative}${P_ERROR}'.${P_END}"
+  [ -z "$path_glob" ] && continue;
+
+  path_absolute="$OSCALDIR"/"$path_glob"
+
+  for path in $path_absolute; do
+#    echo "Path: $path"
+#    echo "Format: $format"
+#    echo "Model: $model"
+#    echo "Convert to: $converttoformats"
+
+    paths+=("$path")
+    formats+=("$format")
+    models+=("$model")
+    conversion_formats+=("$converttoformats")
+  done
+done < "${OSCALDIR}/build/ci-cd/config/content"
+IFS="$IFS_OLD"
+
+#echo "Paths: ${paths[@]}"
+#echo "Formats: ${formats[@]}"
+#echo "Models: ${models[@]}"
+#echo "Convert To: ${conversion_formats[@]}"
+
+shopt -s nullglob
+shopt -s globstar
+
+exitcode=0
+for i in ${!paths[@]}; do
+#  echo "Index: $i"
+  source_file="${paths[$i]}"
+  source_format="${formats[$i]}"
+  model="${models[$i]}"
+  converttoformats="${conversion_formats[$i]}"
+
+  # get the base file name
+  source_file_basename=$(basename $source_file)
+  source_file_relative=$(get_rel_path "$OSCALDIR" "$source_file")
+
+  # debuggging statements, shows what is processing
+#  printf 'path: %s\n' "$file"
+#  printf 'file name: %s\n' "$file_basename"
+#  printf 'Source format: %s\n' "$source_format"
+#  printf 'model: %s\n' "$model"
+#  printf 'convert-to: %s\n' "$converttoformats"
+
+  source_schema="$WORKING_DIR/$source_format/schema/oscal_${model}_schema.xsd"
+
+  # copy source to destination
+  # --------------------------
+  target_file="$WORKING_DIR/${source_file/$OSCALDIR\/src\//}"
+  target_dir=${target_file%/*} # remove filename
+  mkdir -p "$target_dir"
+  target_file_relative=$(get_rel_path "${WORKING_DIR}" "$target_file")
+
+  result=$(cp "$source_file" "$target_file" 2>&1)
+  cmd_exitcode=$?
+  if [ $cmd_exitcode -ne 0 ]; then
+    echo -e "${P_ERROR}Unable to copy '${P_END}${source_file_relative}${P_ERROR}' to '${P_END}${target_file_relative}${P_ERROR}'.${P_END}"
+    echo -e "${P_ERROR}${result}${P_END}"
+  else
+    echo -e "${P_OK}Copied '${P_END}${source_file_relative}${P_OK}' to '${P_END}${target_file_relative}${P_OK}'.${P_END}"
+  fi
+
+  #split on commas
+  IFS_OLD="$IFS"
+  IFS=, to_formats=($converttoformats)
+  IFS="$IFS_OLD"
+  for target_format in ${to_formats[@]}; do
+    if [ -z "$target_format" ]; then
+      # skip blanks
+      continue;
+    fi
+
+    # convert to target format
+    # ------------------------
+    newpath="${source_file/$OSCALDIR\/src\//}" # strip off src
+    newpath="${newpath/\/$source_format\///$target_format/}" # change path from old to new format dir
+    newpath="${newpath%.*}" # strip extension
+
+    case $target_format in
+    xml)
+      target_file="$WORKING_DIR/${newpath}-min.${target_format}"
+      ;;
+    json)
+      target_file="$WORKING_DIR/${newpath}-min.${target_format}"
+      ;;
+    *)
+      echo -e "${P_WARN}Conversion from '${source_format} to '${target_format^^}' is unsupported for '${P_END}${source_file_relative}${P_OK}'.${P_END}"
+      continue;
+    esac
+
+    target_dir=${target_file%/*} # remove filename
+    mkdir -p "$target_dir"
+
+    target_file_relative=$(get_rel_path "${WORKING_DIR}" "$target_file")
+    if [ "$VERBOSE" = "true" ]; then
+      echo -e "${P_INFO}Performing ${source_format^^}->${target_format^^} conversion of '${P_END}${source_file_relative}${P_INFO}' to '${P_END}${target_file_relative}${P_INFO}'.${P_END}"
+    fi
+
+    result=$(convert_to_format_and_validate "$source_file" "$target_file" "$source_format" "$target_format" "$model")
+    if [ -n "$result" ]; then
+      echo -e "${result}"
+    fi
+    cmd_exitcode=$?
+    if [ $cmd_exitcode != 0 ]; then
+        exitcode=1
+        continue;
+    else
+      echo -e "${P_OK}Converted ${source_format^^} '${P_END}${source_file_relative}${P_OK}' to ${target_format^^} as '${P_END}${target_file_relative}${P_OK}'.${P_END}"
+    fi
+
+    # Format specific post-processing
+    case $target_format in
+    json)
+      # Remove extra slashes
+      # perl -pi -e 's,\\/,/,g' "${dest}"
+      # translate OSCAL mime types
+      perl -pi -e 's,(application/oscal\.[a-z]+\+)xml\",\1json\",g' "${target_file}"
+      # relative content paths
+      # translate path names, starting first with the xml directory, then the filename
+      perl -pi -e 's,(\.\./[^\"]+(?=/xml/))/xml/,\1/json/,g' "${target_file}"
+      perl -pi -e 's,(\.\./[^\"]+(?=/json/)[^\"]+(?=.xml\")).xml\",\1.json\",g' "${target_file}"
+      perl -pi -e 's,(\"[^/\"]+(?=\.xml\")).xml\",\1.json\",g' "${target_file}"
+
+      # produce pretty JSON
+      target_file_pretty="$WORKING_DIR/${newpath}.${target_format}"
+      target_file_pretty_relative=$(get_rel_path "${WORKING_DIR}" "$target_file_pretty")
+      result=$(jq . "$target_file" > "$target_file_pretty" 2>&1)
+      if [ $? -ne 0 ]; then
         echo -e "${P_ERROR}${result}${P_END}"
-      else
-        echo -e "${P_OK}Copied '${P_END}${file_relative}${P_OK}' to '${P_END}${dest_relative}${P_OK}'.${P_END}"
+        echo -e "${P_ERROR}Unable to execute jq on '${P_END}${target_file_pretty_relative}${P_ERROR}'.${P_END}"
+        exitcode=1
+        continue
       fi
 
-      IFS=","
-      for altformat in "$converttoformats"; do
-        newpath="${file/$OSCALDIR\/src\//}" # strip off src
-        newpath="${newpath/\/$format\///$altformat/}" # change path from old to new format dir
-        newpath="${newpath%.*}" # strip extension
-
-        converter="$WORKING_DIR/$altformat/convert/oscal_${model}_${format}-to-${altformat}-converter.xsl"
-        converter_relative=$(realpath --relative-to="${WORKING_DIR}" "$converter")
-
-
-        case $format in
-        xml)
-          dest="$WORKING_DIR/${newpath}-min.${altformat}"
-          dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest")
-          if [ "$VERBOSE" = "true" ]; then
-            echo -e "${P_INFO}Generating ${altformat^^} file '${P_END}${dest_relative}${P_INFO}' from '${P_END}${file_relative}${P_INFO}' using converter '${P_END}${converter_relative}${P_INFO}'.${P_END}"
-          fi
-          result=$(xsl_transform "$converter" "$file" "$dest" 2>&1)
-          ;;
-        json)
-          dest="$WORKING_DIR/${newpath}.${altformat}"
-          dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest")
-          if [ "$VERBOSE" = "true" ]; then
-            echo -e "${P_INFO}Generating ${altformat^^} file '${P_END}${dest_relative}${P_INFO}' from '${P_END}${file_relative}${P_INFO}' using converter '${P_END}${converter_relative}${P_INFO}'.${P_END}"
-          fi
-          result=$(xsl_transform "$converter" "" "$dest" "-it" "json-file=${file}" 2>&1)
-          ;;
-        *)
-          echo -e "${P_WARN}Conversion from '${format} to '${altformat^^}' is unsupported for '${P_END}${file_relative}${P_OK}'.${P_END}"
-          continue;
-        esac
-        cmd_exitcode=$?
-        if [ $cmd_exitcode -ne 0 ]; then
-          echo -e "${P_ERROR}Content conversion to ${altformat^^} failed for '${P_END}${file_relative}${P_ERROR}' using converter '${P_END}${converter_relative}${P_ERROR}'.${P_END}"
-          echo -e "${P_ERROR}${result}${P_END}"
-          exitcode=1
-          continue
-        else
-          if [ "$VERBOSE" = "true" ]; then
-            echo -e "${P_OK}Content conversion to ${altformat^^} succeeded for '${P_END}${file_relative}${P_OK}'.${P_END}"
-          fi
-        fi
-
-        # Format specific post-processing
-        case $altformat in
-        json)
-          # Remove extra slashes
-          # perl -pi -e 's,\\/,/,g' "${dest}"
-          # translate OSCAL mime types
-          perl -pi -e 's,(application/oscal\.[a-z]+\+)xml\",\1json\",g' "${dest}"
-          # relative content paths
-          # translate path names, starting first with the xml directory, then the filename
-          perl -pi -e 's,(\.\./[^\"]+(?=/xml/))/xml/,\1/json/,g' "${dest}"
-          perl -pi -e 's,(\.\./[^\"]+(?=/json/)[^\"]+(?=.xml\")).xml\",\1.json\",g' "${dest}"
-          perl -pi -e 's,(\"[^/\"]+(?=\.xml\")).xml\",\1.json\",g' "${dest}"
-
-#          cp "${dest}.tmp" "${dest}"
-
-          # validate generated file
-          schema="$WORKING_DIR/json/schema/oscal_${model}_schema.json"
-          schema_relative=$(realpath --relative-to="${WORKING_DIR}" "$schema")
-          result=$(validate_json "$schema" "$dest")
-          cmd_exitcode=$?
-          if [ $cmd_exitcode -ne 0 ]; then
-            echo -e "${P_ERROR}JSON Schema validation failed for '${P_END}${dest_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
-            echo -e "${P_ERROR}${result}${P_END}"
-            exitcode=1
-            continue
-          else
-            echo -e "${P_OK}JSON Schema validation passed for '${P_END}${dest_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
-          fi
-
-          # produce pretty JSON
-          dest_pretty="$WORKING_DIR/${newpath}.${altformat}"
-          dest_pretty_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest_pretty")
-          result=$(jq . "$dest" > "$dest_pretty" 2>&1)
-          if [ $? -ne 0 ]; then
-            echo -e "${P_ERROR}Unable to execute jq on '${P_END}${dest_pretty_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
-            echo -e "${P_ERROR}${result}${P_END}"
-            exitcode=1
-            continue
-          fi
-		  # remove carriage returns
-		  perl -pi -e 's,\r,,g' "$dest_pretty"
-
-          result=$(validate_json "$schema" "$dest_pretty" 2>&1)
-          cmd_exitcode=$?
-          if [ $cmd_exitcode -ne 0 ]; then
-            echo -e "${P_ERROR}JSON Schema validation failed for '${P_END}${dest_pretty_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
-            echo -e "${P_ERROR}${result}${P_END}"
-            exitcode=1
-            continue
-          else
-            echo -e "${P_OK}JSON Schema validation passed for '${P_END}${dest_pretty_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
-          fi
-
-          # produce yaml
-          newpath="${newpath/\/json\///yaml/}" # change path
-          dest_pretty="$WORKING_DIR/${newpath}.yaml"
-          dest_pretty_dir=${dest_pretty%/*} # remove filename
-          mkdir -p "$dest_pretty_dir"
-          prettyjson --nocolor=1 --indent=2 --inline-arrays=1 "$dest" > "$dest_pretty"
-          ;;
-        xml)
-          # validate generated file
-          schema="$WORKING_DIR/xml/schema/oscal_${model}_schema.xsd"
-          schema_relative=$(realpath --relative-to="${WORKING_DIR}" "$schema")
-          result=$(xmllint --noout --schema "$schema" "$dest" 2>&1)
-          cmd_exitcode=$?
-          if [ $cmd_exitcode -ne 0 ]; then
-            echo -e "${P_ERROR}XML Schema validation failed for '${P_END}${dest_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
-            echo -e "${P_ERROR}${result}${P_END}"
-            exitcode=1
-          else
-            echo -e "${P_OK}XML Schema validation passed for '${P_END}${dest_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
-          fi
-          ;;
-        *)
-          echo -e "${P_WARN}Post processing of '${altformat^^}' is unsupported for '${P_END}${dest_relative}${P_OK}'.${P_END}"
-          continue;
-        esac
-      done
-    done
-  fi
-done < "$OSCALDIR/build/ci-cd/config/content"
+      # remove carriage returns
+      perl -pi -e 's,\r,,g' "$target_file_pretty"
+
+      result=$(validate_content "$target_file_pretty" "json" "$model")
+      if [ $? -ne 0 ]; then
+        echo -e "${P_ERROR}${result}${P_END}"
+        echo -e "${P_ERROR}Unable to execute jq on '${P_END}${target_file_pretty_relative}${P_ERROR}'.${P_END}"
+        exitcode=1
+        continue
+      fi
 
-shopt -u nullglob
-shopt -u globstar
+      # produce yaml
+      newpath="${newpath/\/json\///yaml/}" # change path
+      target_file_yaml="$WORKING_DIR/${newpath}.yaml"
+      target_file_yaml_dir=${target_file_yaml%/*} # remove filename
+      mkdir -p "$target_file_yaml_dir"
+      prettyjson --nocolor=1 --indent=2 --inline-arrays=1 "$target_file" > "$target_file_yaml"
+      ;;
+    xml)
+      # do nothing
+      ;;
+    *)
+      echo -e "${P_WARN}Post processing of '${altformat^^}' is unsupported for '${P_END}${dest_relative}${P_OK}'.${P_END}"
+      continue;
+    esac
+  done
+done
 
 exit $exitcode