Implementation refactor plus sprint22 metaschema (#464)

* Supporting enumerated value lists on flags in XSD and JSON Schema #437
* Metaschema improvements and remodeling flags support (local definitions; moving w/in the model; valid values)
* More Metaschema syntax remodeling: now using max-occurs and min-occurs for cardinality indicators #441
* Updating schemas to current model (transitional); updated docs in resulting XSD per #448; some updates to markdown documentation
* Small updates to authoring CSS.
* Added design notes on JSON schema bindings.
* Including an initial set of unit tests for group-by, some data types, etc.
* Updated to new design (#436): Metaschema schema, Schematron, metaschema update XSLT - will break the build for now
* Work on datatyping #436 including support in XML and JSON Schema
* Addressed #451: parameter insertion syntax in Markdown
* Schema documentation improvements #423 #424 #428
* Addressed feature request #438
* Improving handling of JSON 'key' and 'value-key' settings in docs #428
* Now producing XML and JSON model maps from metaschemas.
* Now escaping {{ for Jekyll
* Updated component metaschema based on design notes
* Fixed schematron assertions to properly work with @ref.
* Refactoring of component and SSP metachema.
* Fixed website side navigation. Added component and SSP metaschema to site generation process.
* Addressed comments in issue #445
* Added site content generation to runall.sh
* Started documenting model changes in release notes. More work is still needed to complete this.

.circleci/config.yml

@@ -58,7 +58,7 @@ commands:
             - mvn-m2-cache-V2
       - run:
           name: Get Saxon-HE
-          command: | 
+          command: |
             mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION
       - run:
           name: Build JSON Validation CLI
@@ -158,20 +158,20 @@ executors:
   java-executor:
     docker: # run the java jobs with Docker
       - image: circleci/openjdk:11-jdk-node-browsers
-    <<: *common_environment 
+    <<: *common_environment
   ruby-node-executor:
     docker: # run the ruby jobs with Docker
       - image: circleci/ruby:2.4.6-node-browsers
-    <<: *common_environment 
+    <<: *common_environment
   python-executor:
     docker: #run the python jobs with Docker
       - image: circleci/python:3.7.3-node-browsers
-    <<: *common_environment 
+    <<: *common_environment
   go-executor:
     docker: # run the steps with Docker
       - image: circleci/golang:1.12.5
-    <<: *common_environment 
-jobs: # a collection of 
+    <<: *common_environment
+jobs: # a collection of
   job-ci-skip-check:
     executor: java-executor
     steps:
@@ -287,7 +287,7 @@ jobs: # a collection of
             bash "$CICD_DIR/validate-content.sh" "$OSCAL_BUILD_DIR"
   job-roundtrip-conversions:
     executor: python-executor
-    steps: 
+    steps:
       - *checkout
       - *attach_build_workspace
       - check-skip-build
@@ -347,7 +347,7 @@ jobs: # a collection of
             git add -f --all content
             # check for changes
             echo "Changed files:"
-            
+
             if ! $(git diff --exit-code --name-only HEAD~1 xml json content); then
               # Only deploy if something relevant has changed
               # Setup deployment
@@ -376,10 +376,15 @@ jobs: # a collection of
             cp -vr "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
             cd "$OSCAL_REPO_DIR/docs"
             bundle exec jekyll build -d _site/OSCAL
-            htmlproofer --assume-extension ./_site
+            htmlproofer --allow-hash-href --url-ignore "/\/OSCAL\/docs\/schemas\//,/#/,/https:\/\/raw\.githubusercontent\.com/" --assume-extension --log-level :debug ./_site
           environment:
             JEKYLL_ENV: production
       - *save_gem_cache
+      - run:
+          name: Compress build docs
+          command: |
+            tar -zcvf "${OSCAL_BUILD_DIR}/docs.tar.gz" "${OSCAL_BUILD_DIR}/docs"
+            rm -rf "${OSCAL_BUILD_DIR}/docs"
       - *publish_artifacts
   job-build-and-deploy-website:
     executor: ruby-node-executor

.gitignore

@@ -27,6 +27,7 @@ xspec/*.html
 /xml
 /json
 /svrl
+/metaschema/unit-testing
 /.skipbuild
 /.runbuild
 

README.md

@@ -17,7 +17,7 @@ OSCAL is comprised of multiple components which have been highlighted in the arc
 
 ## Project Status
 
-OSCAL 1.0.0 Milestone 1 was released on June 15, 2019. The full announcement can be find below:
+OSCAL 1.0.0 Milestone 1 was released on June 15, 2019. The full announcement can be found below:
 
 <blockquote>
 We are pleased to announce the release of OSCAL 1.0.0 Milestone 1. This release marks an important milestone for the OSCAL project, as this represents the first official release of OSCAL.
@@ -36,4 +36,4 @@ The OSCAL team will continue the development of OSCAL focusing our full attentio
 Our current experimental OSCAL implementation layer is being validated as part of a pilot with GSA/FedRAMP, to ensure the necessary functionality and enough flexibility is provided to support a wide variety of SSPs provided by different organizations. To further validate the implementation layer's functionality and flexibility, NIST is seeking software and service providers that are willing to work with us to represent control implementation information about their products. Please email us at oscal@nist.gov if you are interested.
 
 If you have any questions about OSCAL in general or the Milestone 1 release specifically, or if you would like to get involved in the OSCAL project, please contact us at: oscal@nist.gov.
-</blockquote>
+</blockquote>

build/ci-cd/README.md

@@ -30,17 +30,19 @@ The bash scripts used in the CI/CD workflow can be run locally in the correct en
 - [Apache Maven](https://maven.apache.org/) 3.6.1 or greater
 - [Node.js and Node Package Manager (NPM)](https://nodejs.org/en/) 10.15.3 or greater
 - 'Another Json Validator' Command Line Interface ([ajv-cli](https://github.com/jessedc/ajv-cli)) v3.0.0 or greater
-- [Python](https://www.python.org/) 3.7.3 or greater
+- [Python](https://www.python.org/) tested with 3.7.3
 
 Additionally, the following packages need to be installed.
 
 NodeJS Packages:
 
 ```
 npm install -g ajv-cli
+```
 
 Python Packages:
 
+```
 pip install lxml
 ```
 
@@ -68,3 +70,111 @@ git clone --depth 1 --no-checkout https://github.com/Schematron/schematron.git "
 cd "$SCHEMATRON_HOME"
 git checkout master -- trunk/schematron/code
 ```
+
+## Further installation notes
+
+Setting up to run in WSL (Windows subsystem for Linux) avoids many Windows-related configuration issues, but requires the entire environment be available from within the subsystem. The easiest way to accomplish this is to install the components into the subsystem directly.
+
+To check whether xmllint is installed:
+
+```
+xmllint -version
+```
+
+To install it
+
+```
+sudo apt install libxml2-utils
+```
+
+To install Java:
+
+```
+sudo apt install openjdk-11-jre-headless
+```
+
+To install NodeJS:
+
+```
+sudo apt install nodejs
+```
+
+And npm (Node package manager, separate from NodeJS):
+
+```
+sudo apt install npm
+```
+
+
+To install AJV (nodeJS JSON validator)
+
+```
+npm install -g ajv-cli
+```
+
+
+or if you want to install globally (for all users):
+
+```
+sudo npm install -g ajv-cli
+```
+
+Ubuntu comes with Python. To check the version:
+
+```
+python3 -V
+```
+
+Scripts were tested with version 3.7.3 but 3.6.8 (the default in Ubuntu under WSL) should work.
+
+Along with Python you also need a library, for which you need pip (a Python installer):
+
+```
+sudo apt install python-pip
+pip install lxml
+
+```
+
+
+To install Maven:
+
+```
+sudo apt install maven
+```
+
+Then run Maven to acquire Saxon:
+
+
+```
+export SAXON_VERSION=9.9.1-3
+mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION
+```
+
+mkdir $HOME/oscal-oss
+
+export SCHEMATRON_HOME=$HOME/oscal-oss/git-schematron
+git clone --depth 1 --no-checkout https://github.com/Schematron/schematron.git "$SCHEMATRON_HOME"
+cd "$SCHEMATRON_HOME"
+git checkout master -- trunk/schematron/code
+
+
+export OSCAL_TOOLS_DIR=$HOME/oscal-oss/oscal_tools
+git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}"
+cd $OSCAL_TOOLS_DIR/json-cli
+mvn clean install
+
+Finally, export instructions in the preceding must also be copied into .bashrc so they persist in your environment.
+
+prettyson
+
+```
+sudo npm install -g prettyjson
+```
+
+
+jq
+
+```
+sudo apt-get install jq
+```
+

build/ci-cd/config/metaschema

@@ -1,4 +1,7 @@
 # path to metaschema source|generate schema format(s)|generate converter target format(s)|generate docs format(s)
 src/metaschema/oscal_catalog_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_profile_metaschema.xml|xml,json|xml,json|xml,json
-src/metaschema/oscal_ssp_metaschema.xml|xml,json|xml,json|
+src/metaschema/oscal_component_metaschema.xml|xml,json|xml,json|xml,json
+src/metaschema/oscal_ssp_metaschema.xml|xml,json|xml,json|xml,json
+
+

build/ci-cd/copy-and-convert-content.sh

@@ -69,19 +69,21 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
 
           # validate generated file
           schema="$working_dir/json/schema/oscal_${model}_schema.json"
-          validate_json "$schema" "$dest"
+          result=$(validate_json "$schema" "$dest")
           cmd_exitcode=$?
           if [ $cmd_exitcode -ne 0 ]; then
+            echo "${P_ERROR}${result}${P_END}"
             echo "${P_ERROR}Validation of '${dest}' failed.${P_END}"
             exitcode=1
           fi
 
           # produce pretty JSON
           dest_pretty="$working_dir/${newpath}.${altformat}"
           jq . "$dest" > "$dest_pretty"
-          validate_json "$schema" "$dest_pretty"
+          result=$(validate_json "$schema" "$dest_pretty")
           cmd_exitcode=$?
           if [ $cmd_exitcode -ne 0 ]; then
+            echo "${P_ERROR}${result}${P_END}"
             echo "${P_ERROR}Validation of '${dest_pretty}' failed.${P_END}"
             exitcode=1
           fi

build/ci-cd/generate-model-documentation.sh

@@ -17,6 +17,9 @@ echo "${P_INFO}Working in '${P_END}${working_dir}${P_INFO}'.${P_END}"
 # the stylesheet used to generate the documentation
 stylesheet="$OSCALDIR/build/metaschema/xml/produce-and-run-either-documentor.xsl"
 
+# the URL of the github repo where generated artifacts will be accessible
+github_url="https://raw.githubusercontent.com/usnistgov/OSCAL/master"
+
 # the directory to generate the documentation in
 schema_doc_dir="${working_dir}/docs/content/documentation/schemas"
 mkdir -p "$schema_doc_dir" # ensure this directory exists
@@ -37,7 +40,7 @@ while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; d
 
   files_to_process="$OSCALDIR"/"$path"
 
-  IFS= # disable word splitting    
+  IFS= # disable word splitting
   for metaschema in $files_to_process
   do
     filename=$(basename -- "$metaschema")
@@ -61,12 +64,16 @@ while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; d
         # skip blanks
         continue;
       fi
-    
+
       # Run the XSL template for the format
       case $format in
       xml)
+        # determine web location of schema
+        schema_url="${github_url}/xml/schema/${base}_schema.xsd"
         ;;
       json)
+        # determine web location of schema
+        schema_url="${github_url}/json/schema/${base}_schema.json"
         ;;
       *)
         echo "${P_WARN}Generating documentation for '${format^^}' is unsupported for '$metaschema'.${P_END}"
@@ -78,7 +85,8 @@ while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; d
       xsl_transform "$stylesheet_path" "$metaschema_path" "" \
         "target-format=${format}" \
         "example-converter-xslt-path=${converter_path}" \
-        "output-path=${output_path}"
+        "output-path=${output_path}" \
+        "schema-path=${schema_url}"
       cmd_exitcode=$?
       if [ $cmd_exitcode -ne 0 ]; then
         echo "${P_ERROR}Generating ${format^^} model documentation failed for '${P_END}${metaschema_path}${P_ERROR}'.${P_END}"

build/ci-cd/generate-schema.sh

@@ -75,7 +75,7 @@ while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; d
       # xml)
         # TODO: Add support for XML schema validation
       json)
-        validate_json "$OSCALDIR/build/ci-cd/json-schema-schema.json" "$schema"
+        result=$(validate_json "$OSCALDIR/build/ci-cd/json-schema-schema.json" "$schema")
         cmd_exitcode=$?
         ;;
       *)
@@ -85,6 +85,7 @@ while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; d
       esac
 
       if [ $cmd_exitcode -ne 0 ]; then
+        echo "${P_ERROR}${result}${P_END}"
         echo "${P_ERROR}Invalid ${format^^} schema '$schema'.${P_END}"
         exitcode=1
       fi

build/ci-cd/init-validate-json.sh

@@ -36,10 +36,14 @@ validate_json() {
     fi
 
     java -cp "$classpath" gov.nist.oscal.json.JsonCLI "$@" "${extra_params[@]}"
-
-    if [ "$?" -ne 0 ]; then
-        echo "${P_ERROR}Error running JsonCLI.${P_END}"
-        return 3
+    exitcode=$?
+    if [ "$exitcode" -ne 0 ]; then
+        if [ "$exitcode" -gt 1 ]; then
+            echo "${P_ERROR}Error running JsonCLI.${P_END}"
+        else
+            echo "${json_file} is invalid."
+        fi
+        return $exitcode
     fi
     return 0
 }

build/ci-cd/run-all.sh

@@ -74,3 +74,4 @@ echo WORKING DIR        = "${WORKING_DIR}"
 [[ "$PERFORM_GENERATION" == "YES" ]] && ("$CIDIR/generate-content-converters.sh" "$WORKING_DIR" "$@" || ("Failed to generate content converters" && exit 4) )
 [[ "$PERFORM_CONVERSION" == "YES" ]] && ("$CIDIR/copy-and-convert-content.sh" "$WORKING_DIR" "$@" || ("Failed to convert content" && exit 5) )
 [[ "$PERFORM_VALIDATION" == "YES" ]] && ("$CIDIR/validate-round-trips.sh" "$WORKING_DIR" "$@" || ("Failed to validate XML->JSON->XML round-trips" && exit 6) )
+[[ "$PERFORM_GENERATION" == "YES" ]] && ("$CIDIR/generate-model-documentation.sh" "$WORKING_DIR" "$@" || ("Failed to generate website schema model content" && exit 7) )

build/ci-cd/schematron-init.sh

@@ -38,18 +38,18 @@ validate_with_schematron() {
     xsl_transform "$compiled_schematron" "$source_file" "$svrl_result"
     cmd_exitcode=$?
     if [ $cmd_exitcode -ne 0 ]; then
-        echo "${P_ERROR}Processing Schematron '$compiled_schematron' failed for target file '$source_file'${P_END}"
+        echo "Processing Schematron '$compiled_schematron' failed for target file '$source_file'"
         return 3
     fi
     # check if the SVRL result contains errors
     if grep --quiet "failed-assert" "$svrl_result"; then
-        echo "${P_ERROR}The file '$source_file' has the following Schematron errors:"
+        echo "The file '$source_file' has the following Schematron errors:"
         # display the errors
         xsl_transform "$OSCALDIR/build/ci-cd/svrl-to-plaintext.xsl" "$svrl_result"
-        echo -n "${P_END}"
+        echo ""
         return 1
     else
-        echo "${P_OK}File '$source_file' passed Schematron validation.${P_END}"
+        echo "File '$source_file' passed Schematron validation."
     fi
 }