circle.yml

# Okay so maybe everyone else already knows all this, but it took some time
# for Michael and I [Han] to really see how everything fits together.
#
# Basically, what we're doing here is automated browser testing, so CircleCI
# handles the automation, and Sauce Labs handles the browser testing.
# Specifically, Sauce Labs offers a REST API to run tests in browsers in VMs,
# and CircleCI can be configured to listen for git pushes and run local
# servers and call out to REST APIs to test against these local servers.
#
# The flow goes like this:
#   - CircleCI notices/is notified of a git push
#   - they pull and checkout and magically know to install dependencies and shit
#       + https://circleci.com/docs/manually/
#   - their magic works fine for MathQuill's dependencies but to run the tests,
#     it foolishly runs `make test`, what an inconceivable mistake
#   - that's where we come in: `circle.yml` lets us override the test script.
#       + https://circleci.com/docs/configuration/
#   - our `circle.yml` first installs and runs a tunnel to Sauce Labs
#   - and runs `make server`
#   - then it calls out to Sauce Labs' REST API to open browsers that reach
#     back through the tunnel to access test pages on the local server
#       + > Sauce Connect allows you to run a test server within the CircleCI
#         > build container and expose it it (using a URL like `localhost:8080`)
#         > to Sauce Labs’ browsers.
#
#         https://circleci.com/docs/browser-testing-with-sauce-labs/
#
#   - boom testing boom


# this file is based on https://github.com/circleci/sauce-connect/blob/a65e41c91e02550ce56c75740a422bebc4acbf6f/circle.yml
# via https://circleci.com/docs/browser-testing-with-sauce-labs/
#
# then translated from 1.0 to 2.0 with: https://circleci.com/docs/2.0/config-translation/

version: 2
jobs:
  build:
    working_directory: ~/mathquill/mathquill
    parallelism: 1
    shell: /bin/bash --login
    # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
    # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables .
    environment:
      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
    # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages.
    # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images.
    # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job.
    # We have selected a pre-built image that mirrors the build environment we use on
    # the 1.0 platform, but we recommend you choose an image more tailored to the needs
    # of each job. For more information on choosing an image (or alternatively using a
    # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
    # To see the list of pre-built images that CircleCI provides for most common languages see
    # https://circleci.com/docs/2.0/circleci-images/
    docker:
    - image: circleci/node:lts-browsers
    steps:
    # Machine Setup
    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
    # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
    - checkout
    # Prepare for artifact and test results  collection equivalent to how it was done on 1.0.
    # In many cases you can simplify this from what is generated here.
    # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
    # Dependencies
    #   This would typically go in either a build or a build-and-test job when using workflows
    # Restore the dependency cache
    - restore_cache:
        keys:
        # This branch if available
        - v1-dep-{{ .Branch }}-
        # Default branch if not
        - v1-dep-master-
        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
        - v1-dep-
    # This is based on your 1.0 configuration file or project settings
    - run:
        command: |-
          # SauceConnect: download if not cached, and launch with retry
          test $SAUCE_USERNAME && test $SAUCE_ACCESS_KEY || {
            echo 'Sauce Labs credentials required. Sign up here: https://saucelabs.com/opensauce/'
            exit 1
          }

          mkdir -p ~/sauce-connect
          cd ~/sauce-connect

          if [ -x sc-*-linux/bin/sc ]; then
            echo Using cached sc-*-linux/bin/sc
          else
            time wget https://saucelabs.com/downloads/sc-4.7.1-linux.tar.gz
            time tar -xzf sc-4.7.1-linux.tar.gz
          fi

          time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \
            --readyfile ~/sauce_is_ready
          test -e ~/sauce_was_ready && exit

          echo 'Sauce Connect failed, try redownloading (https://git.io/vSxsJ)'
          rm -rf *
          time wget https://saucelabs.com/downloads/sc-4.7.1-linux.tar.gz
          time tar -xzf sc-4.7.1-linux.tar.gz

          time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \
            --readyfile ~/sauce_is_ready
          test -e ~/sauce_was_ready && exit

          echo 'ERROR: Exited twice without creating readyfile' \
            | tee /dev/stderr > ~/sauce_is_ready
          exit 1
        background: true
    # The following line was run implicitly in your 1.0 builds based on what CircleCI inferred about the structure of your project. In 2.0 you need to be explicit about which commands should be run. In some cases you can discard inferred commands if they are not relevant to your project.
    - run: if [ -z "${NODE_ENV:-}" ]; then export NODE_ENV=test; fi
    - run: export PATH="~/mathquill/mathquill/node_modules/.bin:$PATH"
    - run: npm install
    # Save dependency cache
    - save_cache:
        key: v1-dep-{{ .Branch }}-{{ epoch }}
        paths:
        # This is a broad list of cache paths to include many possible development environments
        # You can probably delete some of these entries
        - vendor/bundle
        - ~/virtualenvs
        - ~/.m2
        - ~/.ivy2
        - ~/.bundle
        - ~/.go_workspace
        - ~/.gradle
        - ~/.cache/bower
        # These cache paths were specified in the 1.0 config
        - ~/sauce-connect
        - ./node_modules
    # Test
    #   This would typically be a build job when using workflows, possibly combined with build
    # This is based on your 1.0 configuration file or project settings
    - run: |-
        # Generate link to Many-Worlds build and add to GitHub Commit Status
        curl -i -X POST https://api.github.com/repos/mathquill/mathquill/statuses/$CIRCLE_SHA1 \
             -u MathQuillBot:$GITHUB_STATUS_API_KEY \
             -d '{
                   "context": "ci/many-worlds",
                   "state": "success",
                   "description": "Try the tests on the Many-Worlds build of this commit:",
                   "target_url": "http://many-worlds.glitch.me/mathquill/mathquill/commit/'$CIRCLE_SHA1'/test/"
                 }'
    # Safari on Sauce can only connect to port 3000, 4000, 7000, or 8000. Edge needs port 7000 or 8000.
    # https://david263a.wordpress.com/2015/04/18/fixing-safari-cant-connect-to-localhost-issue-when-using-sauce-labs-connect-tunnel/
    # https://support.saucelabs.com/customer/portal/questions/14368823-requests-to-localhost-on-microsoft-edge-are-failing-over-sauce-connect
    - run:
        command: PORT=8000 make server
        background: true
    # Wait for tunnel to be ready (`make server` is much faster, no need to wait for it)
    - run: while [ ! -e ~/sauce_is_ready ]; do sleep 1; done; touch ~/sauce_was_ready; test -z "$(<~/sauce_is_ready)"
    # This is based on your 1.0 configuration file or project settings
    - run:
        command: |-
          # Screenshots: capture in the background while running unit tests
          mkdir -p $CIRCLE_TEST_REPORTS/mocha

          # CircleCI expects test results to be reported in an JUnit/xUnit-style XML file:
          #   https://circleci.com/docs/test-metadata/#a-namemochajsamocha-for-nodejs
          # Our unit tests are in a browser, so they can't write to a file, and Sauce
          # apparently truncates custom data in their test result reports, so instead we
          # POST to this trivial Node server on localhost:9000 that writes the body of
          # any POST request to $CIRCLE_TEST_REPORTS/junit/test-results.xml
          node -e '
            require("http").createServer(function(req, res) {
              res.setHeader("Access-Control-Allow-Origin", "*");
              req.pipe(process.stdout);
              req.on("end", res.end.bind(res));
            })
            .listen(9000);
            console.error("listening on http://0.0.0.0:9000/");
          ' 2>&1 >$CIRCLE_TEST_REPORTS/junit/test-results.xml | {
            # ^ note: `2>&1` must precede `>$CIRCLE_TEST_REPORTS/...` because
            # shell redirect is like assignment; if it came after, then both
            # stdout and stderr would be written to `xunit.xml` and nothing
            # would be  piped into here

            head -1 # wait for "listening on ..." to be logged

            # https://circleci.com/docs/environment-variables/
            build_name="CircleCI build #$CIRCLE_BUILD_NUM"
            if [ $CIRCLE_PR_NUMBER ]; then
              build_name="$build_name: PR #$CIRCLE_PR_NUMBER"
              [ "$CIRCLE_BRANCH" ] && build_name="$build_name ($CIRCLE_BRANCH)"
            else
              build_name="$build_name: $CIRCLE_BRANCH"
            fi
            build_name="$build_name @ ${CIRCLE_SHA1:0:7}"
            export MQ_CI_BUILD_NAME="$build_name"

            time { test -d node_modules/wd || npm install wd; }
            time node script/screenshots.js http://localhost:8000/test/visual.html \
              && touch ~/screenshots_are_ready || echo EXIT STATUS $? | tee /dev/stderr > ~/screenshots_are_ready:
          }
        background: true
    - run: |-
        # Unit tests in the browser

        echo '1. Launch tests'
        echo

        # https://circleci.com/docs/environment-variables/
        build_name="CircleCI build #$CIRCLE_BUILD_NUM"
        if [ $CIRCLE_PR_NUMBER ]; then
          build_name="$build_name: PR #$CIRCLE_PR_NUMBER"
          [ "$CIRCLE_BRANCH" ] && build_name="$build_name ($CIRCLE_BRANCH)"
        else
          build_name="$build_name: $CIRCLE_BRANCH"
        fi
        build_name="$build_name @ ${CIRCLE_SHA1:0:7}"

        # "build" and "customData" parameters from:
        #   https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-TestAnnotation
        set -o pipefail
        curl -i -X POST https://saucelabs.com/rest/v1/$SAUCE_USERNAME/js-tests \
             -u $SAUCE_USERNAME:$SAUCE_ACCESS_KEY \
             -H 'Content-Type: application/json' \
             -d '{
                   "name": "Unit tests, Mocha",
                   "build": "'"$build_name"'",
                   "customData": {"build_url": "'"$CIRCLE_BUILD_URL"'"},
                   "framework": "mocha",
                   "url": "http://localhost:8000/test/unit.html?post_xunit_to=http://localhost:9000",
                   "platforms": [["", "Chrome", ""]]
                 }' \
        | tee /dev/stderr | tail -1 > js-tests.json

        echo '2. Wait for tests to finish:'
        echo
        #   > Make the request multiple times as the tests run until the response
        #   > contains `completed: true` to the get the final results.
        # https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods
        while true  # Bash has no do...while >:(
        do
          sleep 5
          curl -i -X POST https://saucelabs.com/rest/v1/$SAUCE_USERNAME/js-tests/status \
               -u $SAUCE_USERNAME:$SAUCE_ACCESS_KEY \
               -H 'Content-Type: application/json' \
               -d @js-tests.json \
          | tee /dev/stderr | tail -1 > status.json

          # deliberately do `... != false` rather than `... == true`
          # because unexpected values should break rather than infinite loop
          [ "$(jq .completed <status.json)" != false ] && break
        done

        echo '3. Exit with non-zero status code if any unit tests failed'
        exit "$(jq '.["js tests"][0].result.failures' <status.json)"
    - run: |-
        # Stitch together screenshots and diff against master

        echo '0. Wait for screenshots to be ready'
        while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done
        test -z "$(<~/screenshots_are_ready)" || exit 1

        echo '1. Stitch together pieces'
        for img in $(ls $CIRCLE_ARTIFACTS/imgs/pieces/); do
          convert $(ls -1 $CIRCLE_ARTIFACTS/imgs/pieces/$img/*.png | sort -n) -append $CIRCLE_ARTIFACTS/imgs/$img.png
        done

        echo '2. Download the latest screenshots from master'
        echo

        artifacts_json="$(curl https://circleci.com/api/v1/project/mathquill/mathquill/latest/artifacts?branch=master)"
        echo
        echo '/latest/artifacts?branch=master:'
        echo
        echo "$artifacts_json"
        echo

        mkdir $CIRCLE_ARTIFACTS/imgs/baseline/
        baseline_imgs="$(echo "$artifacts_json" \
                      | jq -r '.[] | .url + " -o " + .pretty_path' \
                      | grep '\.png$' \
                      | grep -v '_DIFF\.png$' \
                      | grep -vF '/pieces/' \
                      | grep -vF '/baseline/' \
                      | sed "s:\$CIRCLE_ARTIFACTS/imgs/:$CIRCLE_ARTIFACTS/imgs/baseline/:")"
        echo 'Baseline image URLs and files:'
        echo
        echo "$baseline_imgs"
        echo

        test -z "$baseline_imgs" && { echo 'No baseline images to download'; exit; }
        curl $baseline_imgs
        echo

        echo '3.  Generate image diffs'
        echo
        cd $CIRCLE_ARTIFACTS/imgs/
        for file in $(ls *.png); do
          # if evergreen browser, browser version of previous screenshot may not match,
          # so replace previous browser version with glob
          baseline="$(echo baseline/$(echo $file | sed 's/[^_]*_(evergreen)/*/; s/OS_X_.*/OS_X_*.png/' | tee /dev/stderr) | tee /dev/stderr)"
          echo "Number of different pixels from baseline in $file:"
          compare -metric AE $baseline $file ${file/%.png/_DIFF.png}
          echo
        done
        true  # ignore errors like "image widths or heights differ"
    # This is based on your 1.0 configuration file or project settings
    - run: killall --wait sc; true  # wait for Sauce Connect to close the tunnel; ignore errors since it's just cleanup
    # Teardown
    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
    # Save test results
    - store_test_results:
        path: /tmp/circleci-test-results
    # Save artifacts
    - store_artifacts:
        path: /tmp/circleci-artifacts
    - store_artifacts:
        path: /tmp/circleci-test-results