Enhancing XML- and JSON -conversion logic for metaschema constructs

.circleci/config.yml

@@ -0,0 +1,194 @@
+version: 2.1 # use CircleCI 2.1
+references:
+  common_environment: &common_environment
+    environment:
+      REPO_DIR: git-repo
+      REPO_WEBSITE_DIR: git-repo/website
+      BUILD_DIR: build_artifacts
+      TERM: xterm
+      BUNDLER_VERSION: 2.0.2
+      HUGO_VERSION: 0.58.3
+    working_directory: ~/working
+  attach_build_workspace: &attach_build_workspace
+    attach_workspace:
+        at: build_artifacts
+  checkout: &checkout
+    checkout:
+      path: git-repo
+  checkout-submodules: &checkout-submodules
+    run:
+      name: Checkout Submodules
+      command: |
+        cd "$REPO_DIR"
+        git submodule sync --recursive
+        git submodule update --recursive --init
+  publish_artifacts: &publish_artifacts
+    store_artifacts:
+      path: build_artifacts
+  restore_gem_cache: &restore_gem_cache
+    restore_cache:
+      keys:
+        - gem-cache-v10-{{ arch }}-{{ checksum "git-repo/website/Gemfile.lock" }}
+        - gem-cache-v10-{{ arch }}-
+  save_gem_cache: &save_gem_cache
+    save_cache:
+      key: gem-cache-v10-{{ arch }}-{{ checksum "git-repo/website/Gemfile.lock" }}
+      paths:
+        - git-repo/website/vendor/bundle
+commands:
+  install-hugo:
+    description: "Install Hugo"
+    steps:
+      - run:
+          name: Get Hugo
+          command: |
+            wget "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_Linux-64bit.deb"
+      - run:
+          name: Install Hugo
+          command: |
+            sudo apt-get install "./hugo_extended_${HUGO_VERSION}_Linux-64bit.deb"
+  install-markdown-link-check:
+    description: "Install markdown-link-check"
+    steps:
+      - run:
+          name: Get markdown-link-check
+          command: |
+            sudo npm install -g markdown-link-check
+  install-bundler:
+    description: "Install bundler and html-proofer"
+    steps:
+      - run:
+          name: Install with Bundler
+          command: |
+            cd "$REPO_WEBSITE_DIR"
+            gem update --system
+            gem install bundler
+            bundle install --path vendor/bundle --jobs=4 --retry=3 && bundle clean
+  check-skip-build:
+    steps:
+      - run:
+          name: Check if job should be skipped
+          command: |
+            if [[ -f "$BUILD_DIR/.skipbuild" ]]; then
+              echo "Skipping job."
+              circleci step halt
+            fi
+executors:
+  java-executor:
+    docker: # run the java jobs
+      - image: circleci/openjdk:11-jdk-node-browsers
+    <<: *common_environment
+  ruby-node-executor:
+    docker: # run the ruby jobs
+      - image: circleci/ruby:2.6.5-node-browsers
+    <<: *common_environment
+jobs: # a collection of
+  job-ci-skip-check:
+    executor: java-executor
+    steps:
+      - checkout
+      - run:
+          name: Check if skip build
+          command: |
+            mkdir -p "$BUILD_DIR/state"
+            COMMIT_MSG=$(git log --format=oneline -n 1 $CIRCLE_SHA1)
+            if [[ $COMMIT_MSG == *"[ci deploy skip]"* ]]; then
+              echo "[ci deploy skip] detected. Stopping."
+              echo "branch: $CIRCLE_BRANCH" > "$BUILD_DIR/.skipbuild"
+            else
+              echo "branch: $CIRCLE_BRANCH" > "$BUILD_DIR/.runbuild"
+            fi
+      - persist_to_workspace:
+          root: build_artifacts
+          paths:
+            - .runbuild
+            - .skipbuild
+  job-validate-markdown:
+    executor: java-executor
+    steps:
+      - *checkout
+      - *attach_build_workspace
+      - check-skip-build
+      - install-markdown-link-check
+      - run:
+          name: Validate repo Markdown content instances
+          command: |
+            # this command will filter out any docs Markdown files, which are checked in a different job
+            find "$REPO_DIR" -path "${REPO_WEBSITE_DIR}" -prune -o -name \*.md -print0 | xargs -0 -n1 markdown-link-check -q -c "$REPO_DIR/scripts/config/.markdown-link-check/config.json"
+  job-validate-site-content:
+    executor: ruby-node-executor
+    steps:
+      - *checkout
+      - *checkout-submodules
+      - *attach_build_workspace
+      - check-skip-build
+      - *restore_gem_cache
+      - install-bundler
+      - install-hugo
+      - run:
+          name: Build and Validate Hugo site
+          command: |
+            cd "${REPO_WEBSITE_DIR}"
+            hugo -v --debug --minify
+            bundle exec htmlproofer --allow-hash-href --disable-external  public/ --url-swap "\/metaschema\/:/"  --assume-extension --log-level :debug
+      - *save_gem_cache
+      - *publish_artifacts
+  job-build-and-deploy-website:
+    executor: ruby-node-executor
+    steps:
+      - *checkout
+      - *checkout-submodules
+      - *attach_build_workspace
+      - check-skip-build
+      - *restore_gem_cache
+      - install-bundler
+      - install-hugo
+      - run:
+          name: Build Site with Hugo
+          command: |
+            # cp -vr "$BUILD_DIR"/* "$REPO_DIR"
+            cd "$REPO_WEBSITE_DIR"
+            hugo -v --debug --minify
+            mkdir -p public/.circleci
+            # copy the circleci config
+            cp ../.circleci/config.yml public/.circleci/config.yml
+            touch public/.nojekyll
+      - *save_gem_cache
+      - run:
+          name: Run deploy script
+          command: |
+            cd "$REPO_WEBSITE_DIR"
+            git config user.name "Deployment Bot"
+            bash ./deploy.sh --push-only -v -m "Deploying website [ci deploy skip]"
+      - *publish_artifacts
+workflows:
+  version: 2
+  build:
+    jobs:
+      - job-ci-skip-check:
+          filters:
+            tags:
+              ignore: /.*/
+      - job-validate-markdown:
+          requires:
+            - job-ci-skip-check
+          filters:
+            branches:
+              ignore:
+                - master
+                - nist-pages
+      - job-validate-site-content:
+          requires:
+            - job-ci-skip-check
+          filters:
+            branches:
+              # only from PRs
+              ignore:
+                - master
+                - nist-pages
+      - job-build-and-deploy-website:
+          requires:
+            - job-ci-skip-check
+          filters:
+            branches:
+              only: master

.github/CONTENTS.md

@@ -0,0 +1,7 @@
+# OSCAL GitHub Supporting Files
+
+This directory contains supporting files for Metaschema GitHub repository use. The structure and contents of the directory are as follows:
+
+* [ISSUE_TEMPLATE](ISSUE_TEMPLATE): This directory contains several templates for creating new issues in GitHub.
+* [issue_template.md](issue_template.md): This file contains a template for creating a new issue in GitHub containing a user story.
+* [PULL_REQUEST_TEMPLATE.md](issue_template.md): This file contains a template to be filled out for each GitHub pull request by the person submitting it.

.github/ISSUE_TEMPLATE/README.md

@@ -0,0 +1,7 @@
+# OSCAL GitHub Issue Template Files
+
+This directory contains templates for creating new project issues in GitHub. The structure and contents of the directory are as follows:
+
+* [bug_report.md](bug_report.md): This file contains a template for creating a new issue in GitHub to report a bug.
+* [question.md](question.md): This file contains a template for creating a new issue in GitHub containing a general question about the project.
+* [feature_request.md](feature_request.md): This file contains a template for creating a new issue in GitHub containing a user story.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,39 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+# Describe the bug
+
+{A clear and concise description of what the bug is.}
+
+# Who is the bug affecting?
+
+# What is affected by this bug?
+
+{Describe the impact the bug is having.}
+
+# When does this occur?
+
+{Describe the conditions under which the bug is occurring.}
+
+# How do we replicate the issue?
+
+{What are the steps to reproduce the behavior?
+1. Do this...
+1. Then this...
+1. See error
+
+If applicable, add screenshots to help explain your problem.}
+
+# Expected behavior (i.e. solution)
+
+{A clear and concise description of what you expected to happen.}
+
+# Other Comments
+
+{Add any other context about the problem here.}

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,28 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: User Story, enhancement
+assignees: ''
+
+---
+
+# User Story:
+
+As a Metaschema {stakeholder}, I {provide a clear and concise description of what the problem is. Ex. I need to be able to do}
+
+## Goals:
+
+{A clear and concise description of what you want to happen. This should be outcome focused. Include concise description of any alternative solutions or features you've considered. Feel free to include screenshots or examples about the feature request here.}
+
+## Dependencies:
+
+{Describe any previous issues or related work that must be completed to start or complete this issue.}
+
+## Acceptance Criteria
+
+- [ ] All [website](https://pages.nist.gov/metaschema) and readme documentation affected by the changes in this issue have been updated. Changes to the website can be made in the docs/content directory of your branch.
+- [ ] A Pull Request (PR) is submitted that fully addresses the goals of this User Story. This issue is referenced in the PR.
+- [ ] The CI-CD build process runs without any reported errors on the PR. This can be confirmed by reviewing that all checks have passed in the PR.
+
+{The items above are general acceptance criteria for all User Stories. Please describe anything else that must be completed for this issue to be considered resolved.}

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: Question
+about: Ask your question
+title: ''
+labels: question
+assignees: ''
+
+---
+
+{Please enter your question.}

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+# Committer Notes
+
+{Please provide a brief description of what this PR accomplishes. Be sure to reference any issues addressed. If the PR is a work-in-progress submitted for early review, please include [WIP] at the beginning of the title or mark the PR as DRAFT.}
+
+### All Submissions:
+
+- [ ] Have you followed the guidelines in our [Contributing](https://github.com/usnistgov/metaschema/blob/master/CONTRIBUTING.md) document?
+- [ ] Have you checked to ensure there aren't other open [Pull Requests](https://github.com/usnistgov/metaschema/pulls) for the same update/change?
+- [ ] Have you squashed any non-relevant commits and commit messages? \[[instructions](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History)\]
+- [ ] Do all automated CI/CD checks pass?
+
+### Changes to Core Features:
+
+- [ ] Have you added an explanation of what your changes do and why you'd like us to include them?
+- [ ] Have you written new tests for your core changes, as applicable?
+- [ ] Have you included examples of how to use your new feature(s)?
+- [ ] Have you updated all website](https://pages.nist.gov/metaschema) and readme documentation affected by the changes you made? Changes to the website can be made in the website/content directory of your branch.

.github/issue_template.md

@@ -0,0 +1 @@
+{Please describe the nature of your issue}

.gitignore

@@ -2,3 +2,11 @@
 # Files generated by Hugo
 /website/resources
 /website/public
+
+# files generated by bundler
+/website/.bundle
+/website/vendor
+
+
+# other generated files
+nohup.out

CONTRIBUTING.md

@@ -42,7 +42,6 @@ This repository consists of the following directories and files pertaining to th
 [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md): This file contains a code of conduct for all project contributors.
 - [CONTRIBUTING.md](CONTRIBUTING.md): This file is for potential contributors to the project. It provides basic information on the project, describes the main ways people can make contributions, explains how to report issues, and lists pointers to additional sources of information. It also has instructions on establishing a development environment for contributing to the project and using GitHub project cards to track development sprints.
 - [LICENSE.md](LICENSE.md): This file contains license information for the files in this GitHub repository.
-- [USERS.md](USERS.md): This file explains which types of users are most likely to benefit from use of this project and its artifacts.
 
 ## Contributing to a Development Sprint
 

config.sh-default

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copy this file to config.sh in the same directory, and uncomment and customize the variables below to override default configuration options for all scripts.
+
+# The PROVIDER_DIR variable identifies the metaschema framework implementation to use.
+# A valid provider will have an "init.sh" file in the specified directory which implements the following set of bash functions:
+#
+# generate_xml_schema <metaschema_file> <generated_schema_file>
+# generate_json_schema <metaschema_file> <generated_schema_file>
+
+#PROVIDER_DIR="${METASCHEMA_SCRIPT_DIR}/../toolchains/oscal-m2"
+
+# The location to write generated files to
+
+#WORKING_DIR="${PWD}"
+
+# The location to cache long-lived files used by scripts
+
+#CACHE_DIR="${WORKING_DIR}/.metaschema-cache"
+
+# Controls if scripts should produce verbose output
+
+#VERBOSE=false
+

scripts/config/.markdown-link-check/config.json

@@ -0,0 +1,10 @@
+{
+  "ignorePatterns": [
+    {
+      "pattern": "^../../issues"
+    },
+    {
+      "pattern": "^../../projects"
+    }
+  ]
+}