Convert File Type for Files or Remote Hyperlinks in Continuous Deploy…

…ment (#1070)

* Convert File Type for Files or Remote Hyperlinks in Continuous Deployment (#1010)

* First attempt at Python script to clean up extensions.
* Add Python deps manifest.
* Update Dockerfile with Python dependencies.
* Remove additive dot before file ext and try out adding to CI harness.
* Add first attempt of Python converter.
* updated the tools table
* Rename handler function accordingly, so docs make sense.
* Fix bug with hard-coded conversion for JSON processing.
* Add some more function documentation.
* Complete update of function docstrings.
* A little more function documentation.
* Final function signature touch-ups.

Co-authored-by: Michaela Iorga <michaela.iorga@nist.gov>
Co-authored-by: David Waltermire <david.waltermire@nist.gov>

* fix python calls to use correct python version and script

Co-authored-by: Alexander Stein (Inactive) <61464190+ohsh6o@users.noreply.github.com>
Co-authored-by: Michaela Iorga <michaela.iorga@nist.gov>

build/Dockerfile

@@ -1,5 +1,7 @@
 FROM ubuntu:20.04 as base-environment
 
+ADD ./ci-cd/python/requirements.txt .
+
 ENV TZ=US/Eastern
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
@@ -15,7 +17,7 @@ RUN apt-get install -y apt-utils build-essential git jq libxml2-utils maven node
 RUN npm install -g npm n
 RUN n latest
 RUN npm install --loglevel verbose -g ajv-cli@"^4.0.x" ajv-formats@"^1.5.x" json-diff markdown-link-check yaml-convert@"^1.0.x" yargs
-RUN pip3 install lxml
+RUN pip3 install -r requirements.txt
 
 #RUN useradd --create-home --home-dir /home/user user
 #USER user

build/ci-cd/copy-and-convert-content.sh

@@ -154,13 +154,7 @@ post_process_content() {
       if [ "$VERBOSE" = "true" ]; then
         echo -e "${P_INFO}Translating relative XML paths to JSON paths in '${P_END}${target_file_relative}${P_INFO}'.${P_END}"
       fi
-      # Remove extra slashes
-      perl -pi -e 's,\\/,/,g' "${target_file}"
-      # translate OSCAL mime types
-      perl -pi -e 's,(application/(oscal\.)?[a-z]+\+)xml\",\1json\",g' "${target_file}"
-      # relative content paths
-      # translate path names for local references
-      perl -pi -e 's,((?:\.\./)+(?:(?!xml/)[^\s/"'']+/)+)xml/((?:(?!.xml)[^\s"'']+)+).xml,\1json/\2.json,g' "${target_file}"
+      python3 "$OSCALDIR/build/ci-cd/python/convert_filetypes.py" --old-extension xml --new-extension json "${target_file}"
     fi
 
     # produce pretty JSON
@@ -205,11 +199,7 @@ post_process_content() {
     if [ "$VERBOSE" = "true" ]; then
       echo -e "${P_INFO}Translating relative paths in '${P_END}${yaml_file_relative}${P_INFO}'.${P_END}"
     fi
-    # translate OSCAL mime types
-    perl -pi -e 's,(application/oscal\.[a-z]+\+)json\",\1yaml\",g' "${yaml_file}"
-    # translate path names for local references
-    perl -pi -e 's,((?:\.\./)+(?:(?!json/)[^\s/"'']+/)+)json/((?:(?!.json)[^\s"'']+)+).json,\1yaml/\2.yaml,g' "${yaml_file}"
-
+    python3 "$OSCALDIR/build/ci-cd/python/convert_filetypes.py" --old-extension json --new-extension yaml "${yaml_file}"
     echo -e "${P_OK}Created YAML '${P_END}${yaml_file_relative}${P_OK}'.${P_END}"
     ;;
   xml)