Model refactoring based on feedback (Session 1)

.circleci/config.yml

@@ -6,7 +6,8 @@ references:
       SAXON_VERSION: 9.9.0-1
       JSON_CLI_VERSION: 0.0.1-SNAPSHOT
       CICD_DIR: git-oscal/build/ci-cd
-      OSCAL_REPO_DIR: git-oscal
+      REPO_DIR: git-oscal
+      REPO_WEBSITE_DIR: git-oscal/docs
       OSCAL_BUILD_DIR: build_artifacts
       OSCAL_TOOLS_DIR: oscal_tools
       SCHEMATRON_HOME: git-schematron
@@ -23,7 +24,7 @@ references:
     run:
       name: Checkout Submodules
       command: |
-        cd "$OSCAL_REPO_DIR"
+        cd "$REPO_DIR"
         git submodule sync --recursive
         git submodule update --recursive --init
   publish_artifacts: &publish_artifacts
@@ -128,6 +129,13 @@ commands:
           name: Get ajv
           command: |
             sudo npm install -g ajv-cli
+  install-json-diff:
+    description: "Install json-diff"
+    steps:
+      - run:
+          name: Get json-diff
+          command: |
+            sudo npm install -g json-diff
   install-markdown-link-check:
     description: "Install markdown-link-check"
     steps:
@@ -162,7 +170,7 @@ commands:
       - run:
           name: Install with Bundler
           command: |
-            cd "$OSCAL_REPO_DIR/docs"
+            cd "$REPO_WEBSITE_DIR"
             gem update --system
             gem install bundler
             bundle install --path vendor/bundle --jobs=4 --retry=3 && bundle clean
@@ -217,6 +225,7 @@ jobs: # a collection of
     executor: java-executor
     steps:
       - *checkout
+      - *checkout-submodules
       - *attach_build_workspace
       - check-skip-build
       - install-xmllint
@@ -235,6 +244,7 @@ jobs: # a collection of
     executor: java-executor
     steps: # a collection of executable commands
       - *checkout
+      - *checkout-submodules
       - *attach_build_workspace
       - check-skip-build
       - install-maven-dependencies
@@ -253,6 +263,7 @@ jobs: # a collection of
     executor: java-executor
     steps: # a collection of executable commands
       - *checkout
+      - *checkout-submodules
       - *attach_build_workspace
       - check-skip-build
       - install-maven-dependencies
@@ -270,6 +281,7 @@ jobs: # a collection of
     executor: java-executor
     steps: # a collection of executable commands
       - *checkout
+      - *checkout-submodules
       - *attach_build_workspace
       - check-skip-build
       - install-maven-dependencies
@@ -289,18 +301,21 @@ jobs: # a collection of
     executor: java-executor
     steps:
       - *checkout
+      - *checkout-submodules
       - *attach_build_workspace
       - check-skip-build
       - install-markdown-link-check
       - run:
           name: Validate repo Markdown content instances
           command: |
+            cd "$REPO_DIR"
             # this command will filter out any docs Markdown files, which are checked in a different job
-            find "$OSCAL_REPO_DIR" -path "${OSCAL_REPO_DIR}/docs" -prune -o -name \*.md -print0 | xargs -0 -n1 markdown-link-check -q
+            git ls-files "*/*.md" -z | grep --null-data -v "^docs/" | xargs -0 -n1 markdown-link-check -q -c "build/config/.markdown-link-check/config.json"
   job-validate-content:
     executor: java-executor
     steps:
       - *checkout
+      - *checkout-submodules
       - *attach_build_workspace
       - check-skip-build
       - install-xmllint
@@ -313,12 +328,14 @@ jobs: # a collection of
     executor: python-executor
     steps:
       - *checkout
+      - *checkout-submodules
       - *attach_build_workspace
       - check-skip-build
       - install-maven
       - install-maven-dependencies
       - install-lxml
       - install-xmllint
+      - install-json-diff
       - run:
           name: Generate round trip XML and JSON conversions
           command: |
@@ -354,7 +371,7 @@ jobs: # a collection of
       - deploy: # delete and copy artifacts before deploy
           name: Commit Artifacts
           command: |
-            cd "$OSCAL_REPO_DIR"
+            cd "$REPO_DIR"
             # Remove existing generated files
             git rm -r --ignore-unmatch xml/convert/*.xsl
             git rm -r --ignore-unmatch xml/schema/*.xsd
@@ -365,8 +382,8 @@ jobs: # a collection of
             git rm -r --ignore-unmatch content/**/*.yaml
             # Copy new built files to repo
             cd -
-            cp -r "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
-            cd "$OSCAL_REPO_DIR"
+            cp -r "$OSCAL_BUILD_DIR"/* "$REPO_DIR"
+            cd "$REPO_DIR"
             # add the new files
             git add -f --all xml
             git add -f --all json
@@ -402,14 +419,14 @@ jobs: # a collection of
           name: Restore generated site content
           command: |
             # restore docs from tar
-            tar -zxvf "${OSCAL_BUILD_DIR}/docs.tar.gz" --directory "${OSCAL_REPO_DIR}/docs" --strip-components=2
+            tar -zxvf "${OSCAL_BUILD_DIR}/docs.tar.gz" --directory "${REPO_WEBSITE_DIR}" --strip-components=2
       - run:
           name: Build and Validate Hugo site
           command: |
-            # cp -vr "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
-            cd "$OSCAL_REPO_DIR/docs"
+            # cp -vr "$OSCAL_BUILD_DIR"/* "$REPO_DIR"
+            cd "$REPO_WEBSITE_DIR"
             hugo -v --debug --minify
-            bundle exec htmlproofer --allow-hash-href --disable-external  public/ --url-swap "\/OSCAL\/:/"  --assume-extension --log-level :debug --url-ignore "/\/OSCAL\/docs\/schemas\//,/#/,/https:\/\/raw\.githubusercontent\.com/"
+            bundle exec htmlproofer --allow-hash-href public/ --url-swap "https\://pages.nist.gov/OSCAL/:/" --assume-extension --log-level :debug --url-ignore "/\/OSCAL\/docs\/\/documentation\/schemas\/[^/]+\/(?:json|xml)-(?:model-map|schema)/,/#/,/pages.nist.gov\/(?:nist-header-footer|leaveNotice)\/.+/"
       - *save_gem_cache
       - *publish_artifacts
   job-build-and-deploy-website:
@@ -426,12 +443,12 @@ jobs: # a collection of
           name: Restore generated site content
           command: |
             # restore docs from tar
-            tar -zxvf "${OSCAL_BUILD_DIR}/docs.tar.gz" --directory "${OSCAL_REPO_DIR}/docs" --strip-components=2
+            tar -zxvf "${OSCAL_BUILD_DIR}/docs.tar.gz" --directory "${REPO_WEBSITE_DIR}" --strip-components=2
       - run:
           name: Build Site with Hugo
           command: |
-            # cp -vr "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
-            cd "$OSCAL_REPO_DIR/docs"
+            # cp -vr "$OSCAL_BUILD_DIR"/* "$REPO_DIR"
+            cd "$REPO_WEBSITE_DIR"
             hugo -v --debug --minify
             mkdir -p public/.circleci
             # copy the circleci config
@@ -442,7 +459,7 @@ jobs: # a collection of
       - run:
           name: Run deploy script
           command: |
-            cd "$OSCAL_REPO_DIR/docs"
+            cd "$REPO_WEBSITE_DIR"
             git config user.name "Deployment Bot"
             bash ./deploy.sh --push-only -v -m "Deploying website [ci deploy skip]"
       - *publish_artifacts
@@ -459,6 +476,7 @@ jobs: # a collection of
     executor: java-executor
     steps:
       - *checkout
+      - *checkout-submodules
       - *attach_build_workspace
       - check-skip-build
       - install-xmllint

.gitignore

@@ -29,6 +29,8 @@ xspec/*.html
 # Produced in CircleCI build
 /.skipbuild
 /.runbuild
+# Generated by scripts
+.metaschema-cache
 # generated by /build/ci-cd/generate-content-converters.sh and /build/ci-cd/generate-schema.sh
 /xml
 /json

.gitmodules

@@ -3,3 +3,7 @@
 	url = git@github.com:usnistgov/hugo-uswds.git
 	branch = master
 	update = rebase
+[submodule "build/metaschema"]
+	path = build/metaschema
+	url = git@github.com:usnistgov/metaschema.git
+	branch = master

build/ci-cd/config/content

@@ -1,6 +1,7 @@
 # path to source|format of source|model of source|format(s) to convert to
 src/content/ssp-example/ssp-example.json|json|ssp|xml
-src/content/fedramp.gov/xml/*catalog.xml|xml|catalog|json
+#src/content/fedramp.gov/xml/FedRAMP_catalog.xml|xml|catalog|json
 src/content/fedramp.gov/xml/*profile.xml|xml|profile|json
 src/content/nist.gov/SP800-53/rev4/xml/*catalog.xml|xml|catalog|json
 src/content/nist.gov/SP800-53/rev4/xml/*profile.xml|xml|profile|json
+

build/ci-cd/config/metaschema

@@ -3,3 +3,4 @@ src/metaschema/oscal_catalog_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_profile_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_component_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_ssp_metaschema.xml|xml,json|xml,json|xml,json
+