Add syntax highlighting to all <pre> contents

This change adds https://github.com/tabatkins/highlighter as a submodule
and updates the build script to use the highlight server when running
wattsi, as documented at
tabatkins/highlighter#6 (comment)

This change reverts the revert made in ffe6f19. It fixes the problem
which made that revert necessary; the fix is to make the PYTHONPATH in
the local environment include the path to the Pygments copy in the
highlighter submodule.

Fixes #169

Relies on whatwg/wattsi#63
Addresses #113
Relates to whatwg/html#3768
and whatwg/whatwg.org#215

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "highlighter"]
+	path = highlighter
+	url = https://github.com/tabatkins/highlighter

README.md

@@ -22,7 +22,7 @@ To build locally, you'll need the following commands installed on your system:
 
 - `curl`, `grep`, `perl`, `unzip`
 
-Optionally, for faster builds, you can install [Wattsi](https://github.com/whatwg/wattsi). If you don't bother with that, the build will use [Wattsi Server](https://github.com/domenic/wattsi-server), which requires an internet connection.
+Optionally, for faster builds, you can install [Wattsi](https://github.com/whatwg/wattsi) and Python 2.7 (necessary for applying syntax highlighting to `pre` contents). If you don't bother with that, the build will use [Wattsi Server](https://github.com/domenic/wattsi-server), which requires an internet connection.
 
 ### Running the build
 

build.sh

@@ -26,6 +26,7 @@ HTML_OUTPUT=${HTML_OUTPUT:-$DIR/output}
 export HTML_OUTPUT
 
 SHA_OVERRIDE=${SHA_OVERRIDE:-}
+HIGHLIGHT_SERVER_URL="http://127.0.0.1:8080"
 
 for arg in "$@"
 do
@@ -366,8 +367,10 @@ function processSource {
   perl .pre-process-tag-omission.pl < "$HTML_TEMP/source-expanded-2" | perl .pre-process-index-generator.pl > "$HTML_TEMP/source-whatwg-complete" # this one could be merged
 
   function runWattsi {
-    # Input arguments: $1 is the file to run wattsi on, $2 is a directory for wattsi to write output
-    # to
+    # Input arguments:
+    # - $1 is the file to run Wattsi on
+    # - $2 is a directory for Wattsi to write output to
+    # - $3 is the URL for the syntax-highlighter server
     # Output:
     # - Sets global variable $WATTSI_RESULT to an exit code (or equivalent, for HTTP version)
     # - $HTML_TEMP/wattsi-output directory will contain the output from wattsi on success
@@ -380,9 +383,9 @@ function processSource {
     if $QUIET; then
       WATTSI_ARGS+=( --quiet )
     fi
-    WATTSI_ARGS+=( "$1" "$HTML_SHA" "$2" "$BUILD_TYPE" "$HTML_CACHE/caniuse.json" "$HTML_CACHE/w3cbugs.csv")
+    WATTSI_ARGS+=( "$1" "$HTML_SHA" "$2" "$BUILD_TYPE" "$HTML_CACHE/caniuse.json" "$HTML_CACHE/w3cbugs.csv" "$HIGHLIGHT_SERVER_URL" )
     if hash wattsi 2>/dev/null; then
-      if [ "$(wattsi --version | cut -d' ' -f2)" -lt "$WATTSI_LATEST" ]; then
+      if [[ "$(wattsi --version | cut -d' ' -f2)" -lt "$WATTSI_LATEST" ]]; then
         echo
         echo "Warning: Your wattsi version is out of date. You should to rebuild an"
         echo "up-to-date wattsi binary from the wattsi sources."
@@ -434,7 +437,12 @@ function processSource {
     fi
   }
 
-  runWattsi "$HTML_TEMP/source-whatwg-complete" "$HTML_TEMP/wattsi-output"
+  PYTHONPATH="$DIR/highlighter/highlighter/pygments:${PYTHONPATH}"
+  export PYTHONPATH
+  "$DIR/highlighter/server.py" &
+  HIGHLIGHT_SERVER_PID=$!
+
+  runWattsi "$HTML_TEMP/source-whatwg-complete" "$HTML_TEMP/wattsi-output" "$HIGHLIGHT_SERVER_URL"
   if [[ "$WATTSI_RESULT" == "0" ]]; then
     if [[ "$LOCAL_WATTSI" != true ]]; then
       "$QUIET" || grep -v '^$' "$HTML_TEMP/wattsi-output.txt" # trim blank lines
@@ -456,6 +464,9 @@ function processSource {
     echo "There were errors. Stopping."
     exit "$WATTSI_RESULT"
   fi
+  kill "$HIGHLIGHT_SERVER_PID"
+  # suppresses 'Terminated: 15 "$DIR/highlighter/server.py"' message
+  wait "$HIGHLIGHT_SERVER_PID" 2>/dev/null || # ignore non-zero exit code
 
   function generateBacklinks {
     perl .post-process-partial-backlink-generator.pl "$HTML_TEMP/wattsi-output/index-$1" > "$2/index.html";

ci-deploy/inside-container.sh

@@ -16,7 +16,8 @@ SERVER_PUBLIC_KEY="ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzd
 HTML_OUTPUT="$(pwd)/output"
 export HTML_OUTPUT
 
-# Environment variables set from outside
+# Note: $TRAVIS_PULL_REQUEST is either a number or false, not true or false.
+# https://docs.travis-ci.com/user/environment-variables/#Default-Environment-Variables
 TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST:-false}
 
 # Build the spec into the output directory
@@ -29,8 +30,6 @@ curl --remote-name --fail https://sideshowbarker.net/nightlies/jar/vnu.jar
 java -Xmx1g -jar vnu.jar --skip-non-html "$HTML_OUTPUT"
 echo ""
 
-# Note: $TRAVIS_PULL_REQUEST is either a number or false, not true or false.
-# https://docs.travis-ci.com/user/environment-variables/#Default-Environment-Variables
 if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
   echo "Skipping deploy for non-master"
   exit 0