Fixing broken and stale links in model documentation. Resolves #968. (#…

…973) * Fixing broken and stale links in model documentation. Resolves #968.
usnistgov · Jun 29, 2021 · 58f7535 · 58f7535
1 parent 9daa454
commit 58f7535
Show file tree

Hide file tree

Showing 282 changed files with 420 additions and 907 deletions.
diff --git a/.github/workflows/metaschema-artifacts.yml b/.github/workflows/metaschema-artifacts.yml
@@ -157,119 +157,119 @@ jobs:
           cd "$OSCAL_HOME"
           # this command will filter out any docs Markdown files, which are checked in a different job
           git ls-files "*/*.md" -z | grep --null-data -v "^docs/" | xargs -0 -n1 markdown-link-check -q -c "build/config/.markdown-link-check/config.json"
-  build-and-publish-website:
-    name: Build and Publish Website
-    needs: build-and-publish-metaschema-artifacts
-    runs-on: ubuntu-18.04
-    steps:
-      # use this for main repo master builds
-      - uses: actions/checkout@v2
-        if: ${{ github.repository == env.HOME_REPO && github.ref == format('refs/heads/{0}',env.HOME_BRANCH) }}
-        with:
-          path: ${{ env.OSCAL_HOME }}
-          submodules: recursive
-          token: ${{ secrets.COMMIT_TOKEN }}
-      - name: Get latest from repo
-        if: ${{ github.repository == env.HOME_REPO && github.ref == format('refs/heads/{0}',env.HOME_BRANCH) }}
-        # This will ensure that any files committed by the previous job, will be updated
-        run: |
-          git pull --ff-only
-        working-directory: ${{ env.OSCAL_HOME }}
-      # use this for pull request builds where checkout uses the builtin token
-      - uses: actions/checkout@v2
-        if: ${{ github.repository != env.HOME_REPO || github.ref != format('refs/heads/{0}',env.HOME_BRANCH) }}
-        with:
-          path: ${{ env.OSCAL_HOME }}
-          submodules: recursive
-      # Setup runtime environment
-      # -------------------------
-      # Operating System packages
-      - name: Update APT package metadata
-        run: |
-          sudo rm -rf /var/lib/apt/lists/* && sudo apt-get update
-      # Java JDK 11
-      - name: Set up JDK 11
-        uses: actions/setup-java@v2
-        with:
-          java-version: 11
-          distribution: 'adopt'
-      - name: Cache Maven packages
-        uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-m2-
-      - name: Get Saxon-HE
-        run: |
-          mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION
-      # Ruby and Bundler
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 2.6 # Not needed with a .ruby-version file
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-          working-directory: ${{ env.OSCAL_WEBSITE_PATH }}
-      # Hugo
-      - name: Get Hugo
-        run: |
-          wget https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_Linux-64bit.deb
-      - name: Install Hugo
-        run: |
-          sudo apt-get install ./hugo_extended_${HUGO_VERSION}_Linux-64bit.deb
-      # Build Artifacts
-      # ---------------
-      - name: Generate specification documentation
-        run: |
-          bash "${OSCAL_CICD_PATH}/generate-specification-documentation.sh"
-      - uses: actions/upload-artifact@v2
-        with:
-          name: specification-docs
-          path: |
-            ${{ env.OSCAL_HOME }}/docs/content/documentation/processing/profile-resolution.html
-          retention-days: 5
-      # job-generate-docs
-      - name: Generate Model Documentation
-        run: |
-          echo "PWD: ${PWD}"
-          echo "OSCAL_HOME: ${OSCAL_HOME}"
-          echo "Github Workspace: ${{ github.workspace }}"
-          echo "Branch: $(git branch --show-current)"
+#   build-and-publish-website:
+#     name: Build and Publish Website
+#     needs: build-and-publish-metaschema-artifacts
+#     runs-on: ubuntu-18.04
+#     steps:
+#       # use this for main repo master builds
+#       - uses: actions/checkout@v2
+#         if: ${{ github.repository == env.HOME_REPO && github.ref == format('refs/heads/{0}',env.HOME_BRANCH) }}
+#         with:
+#           path: ${{ env.OSCAL_HOME }}
+#           submodules: recursive
+#           token: ${{ secrets.COMMIT_TOKEN }}
+#       - name: Get latest from repo
+#         if: ${{ github.repository == env.HOME_REPO && github.ref == format('refs/heads/{0}',env.HOME_BRANCH) }}
+#         # This will ensure that any files committed by the previous job, will be updated
+#         run: |
+#           git pull --ff-only
+#         working-directory: ${{ env.OSCAL_HOME }}
+#       # use this for pull request builds where checkout uses the builtin token
+#       - uses: actions/checkout@v2
+#         if: ${{ github.repository != env.HOME_REPO || github.ref != format('refs/heads/{0}',env.HOME_BRANCH) }}
+#         with:
+#           path: ${{ env.OSCAL_HOME }}
+#           submodules: recursive
+#       # Setup runtime environment
+#       # -------------------------
+#       # Operating System packages
+#       - name: Update APT package metadata
+#         run: |
+#           sudo rm -rf /var/lib/apt/lists/* && sudo apt-get update
+#       # Java JDK 11
+#       - name: Set up JDK 11
+#         uses: actions/setup-java@v2
+#         with:
+#           java-version: 11
+#           distribution: 'adopt'
+#       - name: Cache Maven packages
+#         uses: actions/cache@v2
+#         with:
+#           path: ~/.m2/repository
+#           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+#           restore-keys: ${{ runner.os }}-m2-
+#       - name: Get Saxon-HE
+#         run: |
+#           mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION
+#       # Ruby and Bundler
+#       - uses: ruby/setup-ruby@v1
+#         with:
+#           ruby-version: 2.6 # Not needed with a .ruby-version file
+#           bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+#           working-directory: ${{ env.OSCAL_WEBSITE_PATH }}
+#       # Hugo
+#       - name: Get Hugo
+#         run: |
+#           wget https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_Linux-64bit.deb
+#       - name: Install Hugo
+#         run: |
+#           sudo apt-get install ./hugo_extended_${HUGO_VERSION}_Linux-64bit.deb
+#       # Build Artifacts
+#       # ---------------
+#       - name: Generate specification documentation
+#         run: |
+#           bash "${OSCAL_CICD_PATH}/generate-specification-documentation.sh"
+#       - uses: actions/upload-artifact@v2
+#         with:
+#           name: specification-docs
+#           path: |
+#             ${{ env.OSCAL_HOME }}/docs/content/documentation/processing/profile-resolution.html
+#           retention-days: 5
+#       # job-generate-docs
+#       - name: Generate Model Documentation
+#         run: |
+#           echo "PWD: ${PWD}"
+#           echo "OSCAL_HOME: ${OSCAL_HOME}"
+#           echo "Github Workspace: ${{ github.workspace }}"
+#           echo "Branch: $(git branch --show-current)"
 
-          BRANCH="${GITHUB_REF}"
-          echo "Branch(1): ${BRANCH}"
-          if [[ "$BRANCH" =~ ^refs/pull/.* ]]; then
-            BRANCH="${GITHUB_HEAD_REF}"
-            echo "Branch(2): ${BRANCH}"
-          elif [[ "$BRANCH" =~ ^refs/heads/.* ]]; then
-            BRANCH="${GITHUB_REF#refs/heads/}"
-            echo "Branch(3): ${BRANCH}"
-          fi
-          echo "Branch(4): ${BRANCH}"
-          bash "../build/ci-cd/generate-model-documentation.sh" -b "${BRANCH}"
-        working-directory: ${{ github.workspace }}/${{ env.OSCAL_HOME }}/docs
-#      - uses: actions/upload-artifact@v2
-#        with:
-#          name: model-docs
-#          path: |
-#            ${{ env.OSCAL_HOME }}/docs/layouts/partials/generated
-#          retention-days: 5
-      - name: Run Hugo
-        run: |
-            cd "${OSCAL_WEBSITE_PATH}"
-            hugo -v --debug --minify
-      - uses: actions/upload-artifact@v2
-        with:
-          name: website
-          path: |
-            ${{ env.OSCAL_WEBSITE_PATH }}/public
-          retention-days: 5
-#      - name: Run HTMLProofer
-#        run: |
-#          cd "${OSCAL_WEBSITE_PATH}"
-#          #--allow-hash-href
-#          bundle exec htmlproofer --check-external-hash public/ --url-swap "https\://pages.nist.gov/OSCAL/:/" --assume-extension --log-level :debug --url-ignore "/\/OSCAL\/docs\/\/reference\/[^/]+\/[^/]+\/(?:json|xml)-(?:outline|reference|index|definitions)/,/#/,/pages.nist.gov\/(?:nist-header-footer|leaveNotice)\/.+/"
-#      - name: Run deploy script
-#        if: ${{ github.repository == env.HOME_REPO && github.ref == format('refs/heads/{0}',env.HOME_BRANCH) }}
-#        run: |
-#            cd "$OSCAL_WEBSITE_PATH"
-#            git config user.name "Deployment Bot"
-#            bash ./deploy.sh --push-only -v -m "Deploying website [ci deploy skip]"
+#           BRANCH="${GITHUB_REF}"
+#           echo "Branch(1): ${BRANCH}"
+#           if [[ "$BRANCH" =~ ^refs/pull/.* ]]; then
+#             BRANCH="${GITHUB_HEAD_REF}"
+#             echo "Branch(2): ${BRANCH}"
+#           elif [[ "$BRANCH" =~ ^refs/heads/.* ]]; then
+#             BRANCH="${GITHUB_REF#refs/heads/}"
+#             echo "Branch(3): ${BRANCH}"
+#           fi
+#           echo "Branch(4): ${BRANCH}"
+#           bash "../build/ci-cd/generate-model-documentation.sh" -b "${BRANCH}"
+#         working-directory: ${{ github.workspace }}/${{ env.OSCAL_HOME }}/docs
+# #      - uses: actions/upload-artifact@v2
+# #        with:
+# #          name: model-docs
+# #          path: |
+# #            ${{ env.OSCAL_HOME }}/docs/layouts/partials/generated
+# #          retention-days: 5
+#       - name: Run Hugo
+#         run: |
+#             cd "${OSCAL_WEBSITE_PATH}"
+#             hugo -v --debug --minify
+#       - uses: actions/upload-artifact@v2
+#         with:
+#           name: website
+#           path: |
+#             ${{ env.OSCAL_WEBSITE_PATH }}/public
+#           retention-days: 5
+# #      - name: Run HTMLProofer
+# #        run: |
+# #          cd "${OSCAL_WEBSITE_PATH}"
+# #          #--allow-hash-href
+# #          bundle exec htmlproofer --check-external-hash public/ --url-swap "https\://pages.nist.gov/OSCAL/:/" --assume-extension --log-level :debug --url-ignore "/\/OSCAL\/docs\/\/reference\/[^/]+\/[^/]+\/(?:json|xml)-(?:outline|reference|index|definitions)/,/#/,/pages.nist.gov\/(?:nist-header-footer|leaveNotice)\/.+/"
+# #      - name: Run deploy script
+# #        if: ${{ github.repository == env.HOME_REPO && github.ref == format('refs/heads/{0}',env.HOME_BRANCH) }}
+# #        run: |
+# #            cd "$OSCAL_WEBSITE_PATH"
+# #            git config user.name "Deployment Bot"
+# #            bash ./deploy.sh --push-only -v -m "Deploying website [ci deploy skip]"
diff --git a/build/ci-cd/config/metaschema-docs b/build/ci-cd/config/metaschema-docs
@@ -1,9 +1,9 @@
-# path to metaschema source|archetype|HUGO_MODEL_ID|HUGO_MODEL_NAME|HUGO_SCHEMA_ID
-src/metaschema/oscal_complete_metaschema.xml|complete-reference|complete||complete
-src/metaschema/oscal_catalog_metaschema.xml|model-reference|catalog|Catalog|catalog
-src/metaschema/oscal_profile_metaschema.xml|model-reference|profile|Profile|profile
-src/metaschema/oscal_ssp_metaschema.xml|model-reference|system-security-plan|System Security Plan|ssp
-src/metaschema/oscal_component_metaschema.xml|model-reference|component-definition|Component Definition|component-definition
-src/metaschema/oscal_assessment-plan_metaschema.xml|model-reference|assessment-plan|Assessment Plan|assessment-plan
-src/metaschema/oscal_assessment-results_metaschema.xml|model-reference|assessment-results|Assessment Results|assessment-results
-src/metaschema/oscal_poam_metaschema.xml|model-reference|plan-of-action-and-milestones|Plan of Action and Milestones|poam
+# path to metaschema source|archetype|HUGO_MODEL_ID|HUGO_MODEL_NAME|HUGO_LAYER_ID|HUGO_SCHEMA_ID
+src/metaschema/oscal_complete_metaschema.xml|complete-reference|complete|||complete
+src/metaschema/oscal_catalog_metaschema.xml|model-reference|catalog|Catalog|control|catalog
+src/metaschema/oscal_profile_metaschema.xml|model-reference|profile|Profile|control|profile
+src/metaschema/oscal_ssp_metaschema.xml|model-reference|system-security-plan|System Security Plan|implementation|ssp
+src/metaschema/oscal_component_metaschema.xml|model-reference|component-definition|Component Definition|implementation|component-definition
+src/metaschema/oscal_assessment-plan_metaschema.xml|model-reference|assessment-plan|Assessment Plan|assessment|assessment-plan
+src/metaschema/oscal_assessment-results_metaschema.xml|model-reference|assessment-results|Assessment Results|assessment|assessment-results
+src/metaschema/oscal_poam_metaschema.xml|model-reference|plan-of-action-and-milestones|Plan of Action and Milestones|assessment|poam
diff --git a/build/ci-cd/generate-model-documentation.sh b/build/ci-cd/generate-model-documentation.sh
@@ -184,7 +184,7 @@ if [ ! -d "${doc_path}" ] || [ "$DISABLE_ARCHETYPE_CREATION" = "false" ]; then
 fi
 
 exitcode=0
-while IFS="|" read metaschema_path archetype model_id model_name schema_id || [[ -n "$metaschema_path" ]]; do
+while IFS="|" read metaschema_path archetype model_id model_name layer_id schema_id || [[ -n "$metaschema_path" ]]; do
   [[ "$metaschema_path" =~ ^[[:space:]]*# ]] && continue
   # remove leading space
   metaschema_path="${metaschema_path##+([[:space:]])}"
@@ -194,9 +194,14 @@ while IFS="|" read metaschema_path archetype model_id model_name schema_id || [[
   archetype="${archetype##+([[:space:]])}"
   model_id="${model_id##+([[:space:]])}"
   model_name="${model_name##+([[:space:]])}"
+  layer_id="${layer_id##+([[:space:]])}"
   schema_id="${schema_id##+([[:space:]])}"
 
   metaschema="$OSCAL_DIR"/"$metaschema_path"
+
+  # echo "metaschema: ${metaschema}"
+  [ ! -f "$metaschema" ] && continue;
+
   metaschema_relative=$(realpath --relative-to="$WORKING_DIR" "$metaschema")
 
   filename=$(basename -- "$metaschema")
@@ -213,14 +218,15 @@ while IFS="|" read metaschema_path archetype model_id model_name schema_id || [[
   #echo "model_id='${model_id}'"
   #echo "model_name='${model_name}'"
   #echo "schema_id='${schema_id}'"
+  #echo "layer_id='${layer_id}'"
   #echo "model_path='${model_path}'"
 
   # generate reference documentation
   if [ "$DISABLE_ARCHETYPE_CREATION" = "false" ] || [ ! -d "${model_path}" ]; then
     # build the version folder
     #if [ -d "${model_path}" ] && rm -rf "${doc_path}"
 
-    result=$(cd ${DOCS_DIR};HUGO_REF_TYPE="${TYPE}" HUGO_REF_BRANCH="${BRANCH}" HUGO_REF_VERSION="${VERSION}" HUGO_REF_REVISION="${REVISION}" HUGO_MODEL_NAME="${model_name}" HUGO_MODEL_ID="${model_id}" HUGO_SCHEMA_ID="${schema_id}" hugo new --kind ${archetype} "${model_path}" 2>&1)
+    result=$(cd ${DOCS_DIR};HUGO_REF_TYPE="${TYPE}" HUGO_REF_BRANCH="${BRANCH}" HUGO_REF_VERSION="${VERSION}" HUGO_REF_REVISION="${REVISION}" HUGO_MODEL_NAME="${model_name}" HUGO_MODEL_ID="${model_id}" HUGO_SCHEMA_ID="${schema_id}" HUGO_MODEL_CONCEPTS_URL="/concepts/layer/${layer_id}/${schema_id}/" hugo new --kind ${archetype} "${model_path}" 2>&1)
     cmd_exitcode=$?
     if [ $cmd_exitcode -ne 0 ]; then
       echo -e "${P_ERROR}Generating '${P_END}${model_id}${P_OK}' model page failed for revision '${P_END}${REVISION}${P_ERROR}' on branch '${P_END}${BRANCH}${P_ERROR}'.${P_END}"
@@ -266,6 +272,6 @@ while IFS="|" read metaschema_path archetype model_id model_name schema_id || [[
 
   echo -e "${P_OK}Generated docs for '${P_END}${metaschema_relative}${P_OK}' in '${P_END}${model_path}${P_OK}'.${P_END}"
 
-done <"$OSCAL_DIR/build/ci-cd/config/metaschema-docs"
+done <"${WORKING_DIR}/build/ci-cd/config/metaschema-docs"
 
 exit $exitcode
diff --git a/build/metaschema b/build/metaschema
diff --git a/docs/archetypes/complete-reference/_index.md b/docs/archetypes/complete-reference/_index.md
@@ -15,7 +15,7 @@ sidenav:
 
 This reference combines the information items used in all the OSCAL models into a single, comprehensive view. The following reference documentation is available.
 
-- **[Conceptual Overview]({{ getenv "HUGO_MODEL_CONCEPTS_URL" }}):** Provides a high-level overview of the intended purpose and role of each model in the [OSCAL architecture](/concepts/layer/). Describes each model's purpose, target audience, and key concepts.
+- **[Conceptual Overview](/concepts/layer/):** Provides a high-level overview of the intended purpose and role of each model in the [OSCAL architecture](/concepts/layer/). Describes each model's purpose, target audience, and key concepts.
 - **Outline ([JSON/YAML](json-outline/), [XML](xml-outline/)):** Provides a brief listing of the information items for all OSCAL models organized hierarchically. Useful for understanding the structure and basic syntax of all the OSCAL models in a given format.
 - **Reference ([JSON/YAML](json-reference/), [XML](xml-reference/)):** Provides a detailed description of the structure and syntax for the information items for all OSCAL models organized hierarchically. Useful for understanding how to use information items within all the OSCAL models in the given format.
 - **Index ([JSON/YAML](json-index/), [XML](xml-index/)):** Provides a listing of the information items for all OSCAL models organized by where they are used.

diff --git a/docs/archetypes/model-reference/json-definitions.md b/docs/archetypes/model-reference/json-definitions.md
@@ -10,6 +10,6 @@ toc:
   headingselectors: "h1.toc1, h2.toc2, h3.toc3, h4.toc4, h5.toc5, h6.toc6"
 ---
 
-The following is a reference for the JSON object definitions derived from this model's [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}//src/metaschema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_metaschema.xml).
+The following is a reference for the JSON object definitions derived from the [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}//src/metaschema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_metaschema.xml) for this [model]({{ getenv "HUGO_MODEL_CONCEPTS_URL" }}).
 
 <!-- DO NOT REMOVE. Generated text below -->
diff --git a/docs/archetypes/model-reference/json-index.md b/docs/archetypes/model-reference/json-index.md
@@ -10,6 +10,6 @@ toc:
   headingselectors: "h1.toc1"
 ---
 
-The following is an index of each JSON property used in the [JSON format](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/json/schema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.json) for this model. Each entry in the index lists all uses of the given property in the format, which is linked to the corresponding entry in the [JSON Format Reference](../json-reference/). Each entry also lists the formal name for the definition which is linked to the corresponding JSON definition in the [JSON Format Metaschema Reference](../json-definitions/).
+The following is an index of each JSON property used in the [JSON format](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/json/schema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.json) for this [model]({{ getenv "HUGO_MODEL_CONCEPTS_URL" }}). Each entry in the index lists all uses of the given property in the format, which is linked to the corresponding entry in the [JSON Format Reference](../json-reference/). Each entry also lists the formal name for the definition which is linked to the corresponding JSON definition in the [JSON Format Metaschema Reference](../json-definitions/).
 
 <!-- DO NOT REMOVE. Generated text below -->
diff --git a/docs/archetypes/model-reference/json-outline.md b/docs/archetypes/model-reference/json-outline.md
@@ -44,6 +44,6 @@ aliases:
 {{- end }}
 ---
 
-The following outline is a representation of the [JSON format](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/json/schema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.json) for this model. For each property, the name links to the corresponding entry in the [JSON Format Reference](../json-reference/). The cardinality and data type are also provided for each property where appropriate.
+The following outline is a representation of the [JSON format](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/json/schema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.json) for this [model]({{ getenv "HUGO_MODEL_CONCEPTS_URL" }}). For each property, the name links to the corresponding entry in the [JSON Format Reference](../json-reference/). The cardinality and data type are also provided for each property where appropriate.
 
 <!-- DO NOT REMOVE. Generated text below -->