CI: add diff-shades integration (#2725)

Hopefully this makes it much easier to gauge the impacts of future
changes!

.github/workflows/diff_shades.yml

@@ -0,0 +1,134 @@
+name: diff-shades
+
+on:
+  push:
+    branches: [main]
+    paths-ignore: ["docs/**", "tests/**", "*.md"]
+
+  pull_request:
+    path-ignore: ["docs/**", "tests/**", "*.md"]
+
+  workflow_dispatch:
+    inputs:
+      baseline:
+        description: >
+          The baseline revision. Pro-tip, use `.pypi` to use the latest version
+          on PyPI or `.XXX` to use a PR.
+        required: true
+        default: main
+      baseline-args:
+        description: "Custom Black arguments (eg. -l 79)"
+        required: false
+      target:
+        description: >
+          The target revision to compare against the baseline. Same tip applies here.
+        required: true
+      target-args:
+        description: "Custom Black arguments (eg. -S)"
+        required: false
+
+jobs:
+  analysis:
+    name: analysis / linux
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout this repository (full clone)
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+
+      - name: Install diff-shades and support dependencies
+        run: |
+          python -m pip install pip --upgrade
+          python -m pip install https://github.com/ichard26/diff-shades/archive/stable.zip
+          python -m pip install click packaging urllib3
+          # After checking out old revisions, this might not exist so we'll use a copy.
+          cat scripts/diff_shades_gha_helper.py > helper.py
+          git config user.name "diff-shades-gha"
+          git config user.email "diff-shades-gha@example.com"
+
+      - name: Calculate run configuration & metadata
+        id: config
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >
+          python helper.py config ${{ github.event_name }}
+          ${{ github.event.inputs.baseline }} ${{ github.event.inputs.target }}
+          --baseline-args "${{ github.event.inputs.baseline-args }}"
+
+      - name: Attempt to use cached baseline analysis
+        id: baseline-cache
+        uses: actions/cache@v2.1.7
+        with:
+          path: ${{ steps.config.outputs.baseline-analysis }}
+          key: ${{ steps.config.outputs.baseline-cache-key }}
+
+      - name: Install baseline revision
+        if: steps.baseline-cache.outputs.cache-hit != 'true'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: ${{ steps.config.outputs.baseline-setup-cmd }} && python -m pip install .
+
+      - name: Analyze baseline revision
+        if: steps.baseline-cache.outputs.cache-hit != 'true'
+        run: >
+          diff-shades analyze -v --work-dir projects-cache/
+          ${{ steps.config.outputs.baseline-analysis }} -- ${{ github.event.inputs.baseline-args }}
+
+      - name: Install target revision
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: ${{ steps.config.outputs.target-setup-cmd }} && python -m pip install .
+
+      - name: Analyze target revision
+        run: >
+          diff-shades analyze -v --work-dir projects-cache/
+          ${{ steps.config.outputs.target-analysis }} --repeat-projects-from
+          ${{ steps.config.outputs.baseline-analysis }} -- ${{ github.event.inputs.target-args }}
+
+      - name: Generate HTML diff report
+        run: >
+          diff-shades --dump-html diff.html compare --diff --quiet
+          ${{ steps.config.outputs.baseline-analysis }} ${{ steps.config.outputs.target-analysis }}
+
+      - name: Upload diff report
+        uses: actions/upload-artifact@v2
+        with:
+          name: diff.html
+          path: diff.html
+
+      - name: Upload baseline analysis
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ steps.config.outputs.baseline-analysis }}
+          path: ${{ steps.config.outputs.baseline-analysis }}
+
+      - name: Upload target analysis
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ steps.config.outputs.target-analysis }}
+          path: ${{ steps.config.outputs.target-analysis }}
+
+      - name: Generate summary file (PR only)
+        if: github.event_name == 'pull_request'
+        run: >
+          python helper.py comment-body
+          ${{ steps.config.outputs.baseline-analysis }} ${{ steps.config.outputs.target-analysis }}
+          ${{ steps.config.outputs.baseline-sha }} ${{ steps.config.outputs.target-sha }}
+
+      - name: Upload summary file (PR only)
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@v2
+        with:
+          name: .pr-comment-body.md
+          path: .pr-comment-body.md
+
+        # This is last so the diff-shades-comment workflow can still work even if we
+        # end up detecting failed files and failing the run.
+      - name: Check for failed files in both analyses
+        run: >
+          diff-shades show-failed --check --show-log ${{ steps.config.outputs.baseline-analysis }};
+          diff-shades show-failed --check --show-log ${{ steps.config.outputs.target-analysis }}

.github/workflows/diff_shades_comment.yml

@@ -0,0 +1,47 @@
+name: diff-shades-comment
+
+on:
+  workflow_run:
+    workflows: [diff-shades]
+    types: [completed]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+
+      - name: Install support dependencies
+        run: |
+          python -m pip install pip --upgrade
+          python -m pip install click packaging urllib3
+
+      - name: Get details from initial workflow run
+        id: metadata
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >
+          python scripts/diff_shades_gha_helper.py comment-details
+          ${{github.event.workflow_run.id }}
+
+      - name: Try to find pre-existing PR comment
+        if: steps.metadata.outputs.needs-comment == 'true'
+        id: find-comment
+        uses: peter-evans/find-comment@v1
+        with:
+          issue-number: ${{ steps.metadata.outputs.pr-number }}
+          comment-author: "github-actions[bot]"
+          body-includes: "diff-shades"
+
+      - name: Create or update PR comment
+        if: steps.metadata.outputs.needs-comment == 'true'
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ steps.find-comment.outputs.comment-id }}
+          issue-number: ${{ steps.metadata.outputs.pr-number }}
+          body: ${{ steps.metadata.outputs.comment-body }}
+          edit-mode: replace

.pre-commit-config.yaml

@@ -57,7 +57,7 @@ repos:
     rev: v2.5.1
     hooks:
       - id: prettier
-        exclude: ^Pipfile\.lock
+        exclude: \.github/workflows/diff_shades\.yml
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1

docs/contributing/gauging_changes.md

@@ -40,3 +40,56 @@ If you're running locally yourself to test black on lots of code try:
 ```{program-output} black-primer --help
 
 ```
+
+## diff-shades
+
+diff-shades is a tool similar to black-primer, it also runs _Black_ across a list of Git
+cloneable OSS projects recording the results. The intention is to eventually fully
+replace black-primer with diff-shades as it's much more feature complete and supports
+our needs better.
+
+The main highlight feature of diff-shades is being able to compare two revisions of
+_Black_. This is incredibly useful as it allows us to see what exact changes will occur,
+say merging a certain PR. Black-primer's results would usually be filled with changes
+caused by pre-existing code in Black drowning out the (new) changes we want to see. It
+operates similarly to black-primer but crucially it saves the results as a JSON file
+which allows for the rich comparison features alluded to above.
+
+For more information, please see the [diff-shades documentation][diff-shades].
+
+### CI integration
+
+diff-shades is also the tool behind the "diff-shades results comparing ..." /
+"diff-shades reports zero changes ..." comments on PRs. The project has a GitHub Actions
+workflow which runs diff-shades twice against two revisions of _Black_ according to
+these rules:
+
+|                       | Baseline revision       | Target revision              |
+| --------------------- | ----------------------- | ---------------------------- |
+| On PRs                | latest commit on `main` | PR commit with `main` merged |
+| On pushes (main only) | latest PyPI version     | the pushed commit            |
+
+Once finished, a PR comment will be posted embedding a summary of the changes and links
+to further information. If there's a pre-existing diff-shades comment, it'll be updated
+instead the next time the workflow is triggered on the same PR.
+
+The workflow uploads 3-4 artifacts upon completion: the two generated analyses (they
+have the .json file extension), `diff.html`, and `.pr-comment-body.md` if triggered by a
+PR. The last one is downloaded by the `diff-shades-comment` workflow and shouldn't be
+downloaded locally. `diff.html` comes in handy for push-based or manually triggered
+runs. And the analyses exist just in case you want to do further analysis using the
+collected data locally.
+
+Note that the workflow will only fail intentionally if while analyzing a file failed to
+format. Otherwise a failure indicates a bug in the workflow.
+
+```{tip}
+Maintainers with write access or higher can trigger the workflow manually from the
+Actions tab using the `workflow_dispatch` event. Simply select "diff-shades"
+from the workflows list on the left, press "Run workflow", and fill in which revisions
+and command line arguments to use.
+
+Once finished, check the logs or download the artifacts for local use.
+```
+
+[diff-shades]: https://github.com/ichard26/diff-shades#readme