Merge PR #991: Rootstock updates on 2020-02-28

merges #991

.appveyor.yml

@@ -1,4 +1,7 @@
 # See https://www.appveyor.com/docs/getting-started-with-appveyor-for-linux/
+# Don't build branches with a PR, since their build will be created with the PR itself. 
+# Otherwise there would be two builds -- one for the PR and one for the branch.
+# If you're having issues with getting your PR to build, make sure there are no merge conflicts.
 skip_branch_with_pr: true
 
 # Enable 'Do not build on "Push" events' in the AppVeyor project settings

.github/workflows/manubot.yaml

@@ -0,0 +1,50 @@
+name: Manubot
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+jobs:
+  manubot:
+    name: Manubot
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_PULL_REQUEST_SHA: ${{ github.event.pull_request.head.sha }}
+    steps:
+      - name: Set Environment Variables
+        run: |
+          TRIGGERING_SHA=${GITHUB_PULL_REQUEST_SHA:-$GITHUB_SHA}
+          TRIGGERING_SHA_7=${TRIGGERING_SHA::7}
+          echo "::set-env name=TRIGGERING_SHA_7::$TRIGGERING_SHA_7"
+          echo "TRIGGERING_SHA: $TRIGGERING_SHA"
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+        with:
+          # fetch entire commit history to support get_rootstock_commit
+          fetch-depth: 0
+      - name: Install Environment
+        uses: goanpeca/setup-miniconda@v1
+        with:
+          activate-environment: manubot
+          environment-file: build/environment.yml
+          auto-activate-base: false
+          miniconda-version: 'latest'
+      - name: Build Manuscript
+        shell: bash --login {0}
+        run: bash build/build.sh
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: manuscript-${{ github.run_id }}-${{ env.TRIGGERING_SHA_7 }}
+          path: output
+      - name: Deploy Manuscript
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push' && !github.event.repository.fork
+        env:
+          MANUBOT_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MANUBOT_SSH_PRIVATE_KEY: ${{ secrets.MANUBOT_SSH_PRIVATE_KEY }}
+          CI_BUILD_WEB_URL: https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks
+          CI_JOB_WEB_URL: https://github.com/${{ github.repository }}/runs/${{ github.run_id }}
+        shell: bash --login {0}
+        run: bash ci/deploy.sh

.travis.yml

@@ -15,8 +15,8 @@ cache:
     - ci/cache
 deploy:
   provider: script
-  script: bash -o xtrace ci/deploy.sh
+  script: bash ci/deploy.sh
   skip_cleanup: true
   on:
     branch: master
-    condition: $TRAVIS_EVENT_TYPE = "push"
+    condition: $TRAVIS_EVENT_TYPE = "push" && (-v MANUBOT_SSH_PRIVATE_KEY || "${!encrypted_*}" != "")

README.md

@@ -2,7 +2,8 @@
 
 [![HTML Manuscript](https://img.shields.io/badge/manuscript-HTML-blue.svg)](https://greenelab.github.io/deep-review/)
 [![PDF Manuscript](https://img.shields.io/badge/manuscript-PDF-blue.svg)](https://greenelab.github.io/deep-review/manuscript.pdf)
-[![Build Status](https://travis-ci.org/greenelab/deep-review.svg?branch=master)](https://travis-ci.org/greenelab/deep-review)
+[![GitHub Actions Status](https://github.com/greenelab/deep-review/workflows/Manubot/badge.svg)](https://github.com/greenelab/deep-review/actions)
+[![Travis Build Status](https://travis-ci.org/greenelab/deep-review.svg?branch=master)](https://travis-ci.org/greenelab/deep-review)
 [![Code Climate](https://codeclimate.com/github/greenelab/deep-review/badges/gpa.svg)](https://codeclimate.com/github/greenelab/deep-review)
 
 ## Manuscript description
@@ -91,7 +92,6 @@ The directories are as follows:
 + [`webpage`](webpage) is a directory meant to be rendered as a static webpage for viewing the HTML manuscript.
 + [`build`](build) contains commands and tools for building the manuscript.
 + [`ci`](ci) contains files necessary for deployment via continuous integration.
-  For the CI configuration, see [`.travis.yml`](.travis.yml).
 
 ### Local execution
 
@@ -129,19 +129,17 @@ bash build/autobuild.sh
 
 ### Continuous Integration
 
-[![Build Status](https://travis-ci.org/greenelab/deep-review.svg?branch=master)](https://travis-ci.org/greenelab/deep-review)
-
-Whenever a pull request is opened, Travis CI will test whether the changes break the build process to generate a formatted manuscript.
+Whenever a pull request is opened, CI (continuous integration) will test whether the changes break the build process to generate a formatted manuscript.
 The build process aims to detect common errors, such as invalid citations.
-If your pull request build fails, see the Travis CI logs for the cause of failure and revise your pull request accordingly.
+If your pull request build fails, see the CI logs for the cause of failure and revise your pull request accordingly.
 
-When a commit to the `master` branch occurs (for example, when a pull request is merged), Travis CI builds the manuscript and writes the results to the [`gh-pages`](https://github.com/greenelab/deep-review/tree/gh-pages) and [`output`](https://github.com/greenelab/deep-review/tree/output) branches.
+When a commit to the `master` branch occurs (for example, when a pull request is merged), CI builds the manuscript and writes the results to the [`gh-pages`](https://github.com/greenelab/deep-review/tree/gh-pages) and [`output`](https://github.com/greenelab/deep-review/tree/output) branches.
 The `gh-pages` branch uses [GitHub Pages](https://pages.github.com/) to host the following URLs:
 
 + **HTML manuscript** at https://greenelab.github.io/deep-review/
 + **PDF manuscript** at https://greenelab.github.io/deep-review/manuscript.pdf
 
-For continuous integration configuration details, see [`.travis.yml`](.travis.yml).
+For continuous integration configuration details, see [`.github/workflows/manubot.yaml`](.github/workflows/manubot.yaml) if using GitHub Actions or [`.travis.yml`](.travis.yml) if using Travis CI.
 
 ## License
 

USAGE.md

@@ -12,6 +12,9 @@ Markdown files are identified by their `.md` extension and ordered according to
 For basic formatting, check out the [CommonMark Help](https://commonmark.org/help/) page for an introduction to the formatting options provided by standard markdown.
 In addition, Manubot supports an extended version of markdown, tailored for scholarly writing, which includes [Pandoc's Markdown](https://pandoc.org/MANUAL.html#pandocs-markdown) and the extensions discussed below.
 
+The `content/02.delete-me.md` file in the Rootstock repository shows many of the elements and formatting options supported by Manubot.
+See the [raw markdown](https://gitlab.com/manubot/rootstock/blob/master/content/02.delete-me.md#L) in this file and compare it to the [rendered manuscript](https://manubot.github.io/rootstock/).
+
 Within a paragraph in markdown, single newlines are interpreted as whitespace (same as a space).
 A paragraph's source does not need to contain newlines.
 However, "one paragraph per line" makes the git diff less precise, leading to less granular review commenting, and makes conflicts more likely.
@@ -73,9 +76,15 @@ We recommend always specifying the width of SVG images (even if just `width="100
 
 ### Citations
 
-Manubot supports Pandoc [citations](https://pandoc.org/MANUAL.html#citations) via `pandoc-citeproc`.
-However, Manubot performs automated citation processing and metadata retrieval on in-text citations.
-Therefore, citations must be of the following form: `@source:identifier`, where `source` is one of the options described below.
+Manubot supports Pandoc [citations](https://pandoc.org/MANUAL.html#citations).
+Citations are processed in 3 stages:
+
+1. Pandoc parses the input Markdown to locate citation keys.
+2. The [`pandoc-manubot-cite`](https://github.com/manubot/manubot#pandoc-filter) filter automatically retreives the bibliographic metadata for citation keys.
+3. The [`pandoc-citeproc`](https://github.com/jgm/pandoc-citeproc/blob/master/man/pandoc-citeproc.1.md) filter renders in-text citations and generates styled references.
+
+When using Manubot, citation keys should be formatted like `@source:identifier`,
+where `source` is one of the options described below.
 When choosing which source to use for a citation, we recommend the following order:
 
 1. DOI (Digital Object Identifier), cite like `@doi:10.15363/thinklab.4`.
@@ -106,17 +115,60 @@ Note that multiple citations must be semicolon separated.
 Be careful not to cite the same study using identifiers from multiple sources.
 For example, the following citations all refer to the same study, but will be treated as separate references: `[@doi:10.7717/peerj.705; @pmcid:PMC4304851; @pmid:25648772]`.
 
+Citation keys must adhere to the syntax described in the [Pandoc manual](https://pandoc.org/MANUAL.html#citations):
+
+> The citation key must begin with a letter, digit, or `_`, and may contain alphanumerics, `_`, and internal punctuation characters (`:.#$%&-+?<>~/`).
+
+To evaluate whether a citation key fully matches this syntax, try [this online regex](https://regex101.com/r/mXZyY2/latest).
+If the citation key is not valid, use the [citation tag](#citation-tag) workaround below.
+This is required for citation keys that contain forbidden characters such as `;` or `=` or end with a non-alphanumeric character such as `/`.
+<!-- See [jgm/pandoc#6026](https://github.com/jgm/pandoc/issues/6026) for progress on a more flexible Markdown citation key syntax. -->
+
+Prior to Rootstock commit [`6636b91`](https://github.com/manubot/rootstock/commit/6636b912c6b41593acd2041d34cd4158c1b317fb) on 2020-01-14, Manubot processed citations separately from Pandoc.
+Switching to a Pandoc filter improved reliability on complex documents, but restricted the syntax of citation keys slightly.
+Therefore, users upgrading Rootstock may find some citations become invalid.
+By default, `pandoc-manubot-cite` does not fail upon invalid citations, although this can be changed by adding the following to `metadata.yaml`:
+
+```yaml
+pandoc:
+  manubot-fail-on-errors: True
+```
+
 #### Citation tags
 
-The system also supports citation tags, which are recommended for the following applications:
+The system also supports citation tags, which map from one citation key (an alias) to another.
+Tags are recommended for the following applications:
 
-1. A citation's identifier contains forbidden characters, such as `;` or `=`, or ends with a non-alphanumeric character other than `/`.
-   In these instances, you must use a tag.
+1. A citation's identifier contains forbidden characters, you must use a tag.
 2. A single reference is cited many times.
    Therefore, it might make sense to define a tag, so if the citation updates (e.g. a newer version becomes available), only a single change is required.
 
-Tags should be defined in [`content/citation-tags.tsv`](content/citation-tags.tsv).
-If `citation-tags.tsv` defines the tag `study-x`, then this study can be cited like `@tag:study-x`, for example, `[@tag:Zhou2015_deep_sea]`.
+Tags can be defined using Markdown's [link reference syntax](https://spec.commonmark.org/0.29/#link-reference-definitions) as follows:
+
+```markdown
+Citing a URL containing a `?` character [@tag:my-url].
+Citing a DOI containing parentheses [@doi:my-doi].
+
+[@tag:my-url]: url:https://openreview.net/forum?id=HkwoSDPgg
+[@tag:my-doi]: doi:10.1016/S0022-2836(05)80360-2
+```
+
+This syntax is also used by [`pandoc-url2cite`](https://github.com/phiresky/pandoc-url2cite).
+Make sure to place these link reference definitions in their own paragraphs.
+These paragraphs can be in any of the content Markdown files.
+
+Another method for defining tags is to define `pandoc.citekey-aliases` in `metadata.yaml`:
+
+```yaml
+pandoc:
+  citekey-aliases:
+    tag:my-url: url:https://openreview.net/forum?id=HkwoSDPgg
+    tag:my-doi: doi:10.1016/S0022-2836(05)80360-2
+```
+
+For backwards compatibility, tags can also be defined in `content/citation-tags.tsv`.
+If `citation-tags.tsv` defines the tag `study-x`, then this study can be cited like `@tag:study-x`.
+This method is deprecated.
 
 ## Reference metadata
 
@@ -179,7 +231,7 @@ The following YAML shows the supported key–value pairs for an author:
 ```yaml
 github: dhimmel  # strongly suggested
 name: Daniel S. Himmelstein  # mandatory
-initials: DSH  # strongly suggested
+initials: DSH  # optional
 orcid: 0000-0002-3012-7446  # mandatory
 twitter: dhimmel  # optional
 email: daniel.himmelstein@gmail.com  # suggested

build/README.md

@@ -1,17 +1,20 @@
 # Building the manuscript
 
 [`build.sh`](build.sh) builds the repository.
-`sh build/build.sh` should be executed from the root directory of the repository.
+`bash build/build.sh` should be executed from the root directory of the repository.
 By default, `build.sh` creates HTML and PDF outputs.
 However, setting the `BUILD_PDF` environment variable to `false` will suppress PDF output.
 For example, run local builds using the command `BUILD_PDF=false bash build/build.sh`.
 
 To build a DOCX file of the manuscript, set the `BUILD_DOCX` environment variable to `true`.
 For example, use the command `BUILD_DOCX=true bash build/build.sh`.
-To export DOCX for all Travis builds, set a [Travis environment variable](https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings).
+To export DOCX for all CI builds, set an environment variable (see docs for [GitHub Actions](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-environment-variables) or [Travis CI](https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings)).
 Currently, equation numbers via `pandoc-eqnos` are not supported for DOCX output.
-There is varying support for embedding images in DOCX output.
-Please reference [Pull Request #40](https://github.com/manubot/rootstock/pull/40) for possible solutions and continued discussion.
+
+Format conversion is done using [Pandoc](https://pandoc.org/MANUAL.html).
+`build.sh` calls `pandoc` commands using the options specified in [`pandoc-defaults`](pandoc-defaults).
+Each file specifies a set of pandoc `--defaults` options for a given format.
+To change the options, either edit the YAML files directly or add additional `--defaults` files.
 
 ## Environment
 
@@ -49,7 +52,8 @@ If Docker is available, `build.sh` uses the [Athena](https://www.athenapdf.com/)
 Otherwise, `build.sh` uses [WeasyPrint](https://weasyprint.org/) to build the PDF.
 It is common for WeasyPrint to generate many warnings and errors that can be safely ignored.
 Examples are shown below:
-```
+
+```text
 WARNING: Ignored `pointer-events: none` at 3:16, unknown property.
 WARNING: Ignored `font-display:auto` at 1:53114, descriptor not supported.
 ERROR: Failed to load font at "https://use.fontawesome.com/releases/v5.7.2/webfonts/fa-brands-400.eot#iefix"

build/build.sh

@@ -17,45 +17,22 @@ manubot process \
   --content-directory=content \
   --output-directory=output \
   --cache-directory=ci/cache \
+  --skip-citations \
   --log-level=INFO
 
-# pandoc settings
-CSL_PATH=build/assets/style.csl
-BIBLIOGRAPHY_PATH=output/references.json
-INPUT_PATH=output/manuscript.md
+# Pandoc's configuration is specified via files of option defaults
+# located in the PANDOC_DEFAULTS_DIR directory.
+PANDOC_DEFAULTS_DIR="${PANDOC_DEFAULTS_DIR:-build/pandoc-defaults}"
 
 # Make output directory
 mkdir -p output
 
 # Create HTML output
-# http://pandoc.org/MANUAL.html
+# https://pandoc.org/MANUAL.html
 echo >&2 "Exporting HTML manuscript"
 pandoc --verbose \
-  --from=markdown \
-  --to=html5 \
-  --filter=pandoc-fignos \
-  --filter=pandoc-eqnos \
-  --filter=pandoc-tablenos \
-  --bibliography="$BIBLIOGRAPHY_PATH" \
-  --csl="$CSL_PATH" \
-  --metadata link-citations=true \
-  --include-after-body=build/themes/default.html \
-  --include-after-body=build/plugins/table-scroll.html \
-  --include-after-body=build/plugins/anchors.html \
-  --include-after-body=build/plugins/accordion.html \
-  --include-after-body=build/plugins/tooltips.html \
-  --include-after-body=build/plugins/jump-to-first.html \
-  --include-after-body=build/plugins/link-highlight.html \
-  --include-after-body=build/plugins/table-of-contents.html \
-  --include-after-body=build/plugins/lightbox.html \
-  --include-after-body=build/plugins/attributes.html \
-  --mathjax \
-  --variable math="" \
-  --include-after-body=build/plugins/math.html \
-  --include-after-body=build/plugins/hypothesis.html \
-  --include-after-body=build/plugins/analytics.html \
-  --output=output/manuscript.html \
-  "$INPUT_PATH"
+  --defaults="$PANDOC_DEFAULTS_DIR/common.yaml" \
+  --defaults="$PANDOC_DEFAULTS_DIR/html.yaml"
 
 # Return null if docker command is missing, otherwise return path to docker
 DOCKER_EXISTS="$(command -v docker || true)"
@@ -67,20 +44,9 @@ if [ "${BUILD_PDF:-}" != "false" ] && [ -z "$DOCKER_EXISTS" ]; then
   if [ -L images ]; then rm images; fi  # if images is a symlink, remove it
   ln -s content/images
   pandoc \
-    --from=markdown \
-    --to=html5 \
-    --pdf-engine=weasyprint \
-    --pdf-engine-opt=--presentational-hints \
-    --filter=pandoc-fignos \
-    --filter=pandoc-eqnos \
-    --filter=pandoc-tablenos \
-    --bibliography="$BIBLIOGRAPHY_PATH" \
-    --csl="$CSL_PATH" \
-    --metadata link-citations=true \
-    --webtex=https://latex.codecogs.com/svg.latex? \
-    --include-after-body=build/themes/default.html \
-    --output=output/manuscript.pdf \
-    "$INPUT_PATH"
+    --defaults="$PANDOC_DEFAULTS_DIR/common.yaml" \
+    --defaults="$PANDOC_DEFAULTS_DIR/html.yaml" \
+    --defaults="$PANDOC_DEFAULTS_DIR/pdf-weasyprint.yaml"
   rm images
 fi
 
@@ -112,18 +78,8 @@ fi
 if [ "${BUILD_DOCX:-}" = "true" ]; then
   echo >&2 "Exporting Word Docx manuscript"
   pandoc --verbose \
-    --from=markdown \
-    --to=docx \
-    --filter=pandoc-fignos \
-    --filter=pandoc-eqnos \
-    --filter=pandoc-tablenos \
-    --bibliography="$BIBLIOGRAPHY_PATH" \
-    --csl="$CSL_PATH" \
-    --metadata link-citations=true \
-    --reference-doc=build/themes/default.docx \
-    --resource-path=.:content \
-    --output=output/manuscript.docx \
-    "$INPUT_PATH"
+    --defaults="$PANDOC_DEFAULTS_DIR/common.yaml" \
+    --defaults="$PANDOC_DEFAULTS_DIR/docx.yaml"
 fi
 
 echo >&2 "Build complete"

build/environment.yml

@@ -6,20 +6,21 @@ dependencies:
   - cairocffi=0.8.0
   - cffi=1.12.3
   - ghp-import=0.5.5
-  - jinja2=2.10.3
-  - jsonschema=3.1.1
-  - pandas=0.25.2
-  - pandoc=2.7.3
+  - jinja2=2.11.1
+  - jsonschema=3.2.0
+  - pandas=1.0.1
+  - pandoc=2.9.2
   - pango=1.40.14
-  - pip=19.3
-  - psutil=5.6.3
-  - python=3.7.3
-  - pyyaml=5.1
-  - requests=2.22.0
-  - watchdog=0.9.0
+  - pip=20.0
+  - psutil=5.7.0
+  - python=3.7.6
+  - pyyaml=5.3
+  - requests=2.23.0
+  - watchdog=0.10.2
+  - yamllint=1.20.0
   - pip:
     - errorhandler==2.0.1
-    - git+https://github.com/manubot/manubot@2ffcec1183bdaf7c49a81490b9c31d05e977b351
+    - git+https://github.com/manubot/manubot@a022e715f266c0f116a6b38f993bd696dc62094a
     - jsonref==0.2
     - opentimestamps-client==0.7.0
     - opentimestamps==0.4.1

build/pandoc-defaults/common.yaml

@@ -0,0 +1,13 @@
+# Pandoc --defaults shared between Manubot output formats.
+from: markdown
+input-file: output/manuscript.md
+filters:
+- pandoc-fignos
+- pandoc-eqnos
+- pandoc-tablenos
+- pandoc-manubot-cite
+- pandoc-citeproc
+wrap: preserve
+metadata:
+  csl: build/assets/style.csl
+  link-citations: true