Merge remote-tracking branch 'upstream/8.2.x' into fix.get_uuid_from_…

…db_on_restart

* upstream/8.2.x:
  Update etc/bin/swarm [skip ci]
  GH Actions: attempt FF merge for sync
  GH Actions: always create separate sync branch
  Fix style
  Tweak swarm configure.
  Changelog [skip ci]
  Towncrier: run draft build as part of fast tests
  Bump dev version (#5637)
  Prepare release 8.2.0
  Use towncrier for changelog generation
  Fix changelog [skip ci]
  Tidy CLI help text
  cylc clean remote re-invocation: don't scan for workflows

.github/workflows/1_create_release_pr.yml

@@ -43,15 +43,14 @@ jobs:
           init-file: 'cylc/flow/__init__.py'
           pypi-package-name: 'cylc-flow'
 
-      - name: Update "released on" date in changelog
-        continue-on-error: true
-        uses: cylc/release-actions/stage-1/update-changelog-release-date@v1
-        with:
-          changelog-file: 'CHANGES.md'
-
       - name: Test build
         uses: cylc/release-actions/build-python-package@v1
 
+      - name: Generate changelog
+        run: |
+          python3 -m pip install -q towncrier
+          towncrier build --yes
+
       - name: Create pull request
         uses: cylc/release-actions/stage-1/create-release-pr@v1
         with:

.github/workflows/branch_sync.yml

@@ -4,9 +4,11 @@ on:
   push:
     branches:
       - '8.*.x'
+  schedule:
+    - cron: '33 04 * * 1-5' # 04:33 UTC Mon-Fri
   workflow_dispatch:
     inputs:
-      branch:
+      head_branch:
         description: Branch to merge into master
         required: true
 
@@ -15,26 +17,53 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     env:
-      BRANCH: ${{ inputs.branch || github.ref_name }}
+      HEAD_BRANCH: ${{ inputs.head_branch || github.ref_name }}
+      STATUS_JSON: https://raw.githubusercontent.com/cylc/cylc-admin/master/docs/status/branches.json
     steps:
       - name: Check branch name
         shell: python
         run: |
           import os
+          import json
           import sys
+          from urllib.request import urlopen
+
+          if os.environ['GITHUB_EVENT_NAME'] == 'schedule':
+            # Get branch from status page
+            meta = json.loads(
+              urlopen(os.environ['STATUS_JSON']).read()
+            )['meta_releases']
+            version = min(meta)
+            branch = meta[version][os.environ['GITHUB_REPOSITORY']]
+          else:
+            branch = os.environ['HEAD_BRANCH'].strip()
 
-          branch = os.environ['BRANCH'].strip()
-          if not branch:
-            sys.exit("::error::Branch name cannot be empty")
-          if branch.endswith('deconflict'):
-            sys.exit("::error::Do not run this workflow for already-created deconflict branches")
+          if branch.endswith('-sync'):
+            sys.exit("::error::Do not run this workflow for already-created sync branches")
 
           with open(os.environ['GITHUB_ENV'], 'a') as F:
-            print(f'BRANCH={branch}', file=F)
-            print(f'DECONFLICT_BRANCH={branch}-deconflict', file=F)
+            print(f'HEAD_BRANCH={branch}', file=F)
+            print(f'SYNC_BRANCH={branch}-sync', file=F)
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: master
+
+      - name: Configure git
+        uses: cylc/release-actions/configure-git@v1
+
+      - name: Attempt fast-forward
+        id: ff
+        continue-on-error: true
+        run: |
+          git merge "origin/${HEAD_BRANCH}" --ff-only
+          git push origin master
 
       - name: Check for existing PR
         id: check-pr
+        if: steps.ff.outcome == 'failure'
         shell: python
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -44,7 +73,7 @@ jobs:
           import subprocess
           import sys
 
-          for env_var in ('BRANCH', 'DECONFLICT_BRANCH'):
+          for env_var in ('HEAD_BRANCH', 'SYNC_BRANCH'):
             branch = os.environ[env_var]
             cmd = f'gh pr list -B master -H {branch} -s open --json url -R ${{ github.repository }}'
             ret = subprocess.run(
@@ -55,66 +84,64 @@ jobs:
               print(f"::error::{ret.stderr}")
             if ret.returncode:
               sys.exit(ret.returncode)
-            if json.loads(ret.stdout):
-              print(f"::notice::Found existing PR for {branch}")
+            results: list = json.loads(ret.stdout)
+            if results:
+              print(f"::notice::Found existing PR for {branch} - {results[0]['url']}")
               sys.exit(0)
 
           print("No open PRs found")
           with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
             print('continue=true', file=f)
 
-      - name: Checkout
-        if: steps.check-pr.outputs.continue
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: master
-
-      - name: Configure git
-        if: steps.check-pr.outputs.continue
-        uses: cylc/release-actions/configure-git@v1
-
       - name: Attempt merge
         id: merge
         if: steps.check-pr.outputs.continue
         continue-on-error: true
-        run: git merge "origin/${BRANCH}"
+        run: git merge "origin/${HEAD_BRANCH}"
+
+      - name: Abort merge
+        if: steps.merge.outcome == 'failure'
+        run: git merge --abort
 
       - name: Diff
         id: diff
         if: steps.merge.outcome == 'success'
         run: |
           if [[ "$(git rev-parse HEAD)" == "$(git rev-parse origin/master)" ]]; then
-            echo "::notice::master is up to date with $BRANCH"
+            echo "::notice::master is up to date with $HEAD_BRANCH"
             exit 0
           fi
           if git diff HEAD^ --exit-code --stat; then
-            echo "::notice::No diff between master and $BRANCH"
+            echo "::notice::No diff between master and $HEAD_BRANCH"
             exit 0
           fi
           echo "continue=true" >> $GITHUB_OUTPUT
 
-      - name: Create deconflict branch
-        if: steps.merge.outcome == 'failure'
+      - name: Push sync branch
+        id: push
+        if: steps.merge.outcome == 'failure' || steps.diff.outputs.continue
         run: |
-          git merge --abort
-          git checkout -b "$DECONFLICT_BRANCH" "origin/${BRANCH}"
-          git push origin "$DECONFLICT_BRANCH"
-          echo "BRANCH=${DECONFLICT_BRANCH}" >> $GITHUB_ENV
+          git checkout -b "$SYNC_BRANCH" "origin/${HEAD_BRANCH}"
+          git push origin "$SYNC_BRANCH"
+          echo "continue=true" >> $GITHUB_OUTPUT
 
       - name: Open PR
-        if: steps.merge.outcome == 'failure' || steps.diff.outputs.continue
+        if: steps.push.outputs.continue
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           BODY: |
-            Please do a **normal merge**, not squash merge
+            Please do a **normal merge**, not squash merge.
+            Please fix conflicts if necessary.
 
             ---
 
             Triggered by `${{ github.event_name }}`
         run: |
-          gh pr create --head "$BRANCH" \
-            --title "🤖 Merge ${BRANCH} into master" \
+          url="$(
+            gh pr create --head "$SYNC_BRANCH" \
+            --title "🤖 Merge ${SYNC_BRANCH} into master" \
             --body "$BODY"
+          )"
+          echo "::notice::PR created at ${url}"
 
-          gh pr edit "$BRANCH" --add-label "sync" || true
+          gh pr edit "$SYNC_BRANCH" --add-label "sync" || true

.github/workflows/test_fast.yml

@@ -48,6 +48,10 @@ jobs:
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
+      - name: Check changelog
+        if: startsWith(matrix.os, 'ubuntu')
+        run: towncrier build --draft
+
       - name: Style
         if: startsWith(matrix.os, 'ubuntu')
         run: |

CHANGES.md

@@ -4,13 +4,14 @@ List of notable changes, for a complete list of changes see the
 [closed milestones](https://github.com/cylc/cylc-flow/milestones?state=closed)
 for each release.
 
-<!-- The topmost release date is automatically updated by GitHub Actions. When
-creating a new release entry be sure to copy & paste the span tag with the
-`actions:bind` attribute, which is used by a regex to find the text to be
-updated. Only the first match gets replaced, so it's fine to leave the old
-ones in. -->
+<!--
+NOTE: Do not add entries here, use towncrier fragments instead:
+$ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
+-->
 
-## __cylc-8.2.0 (<span actions:bind='release-date'>Upcoming</span>)__
+<!-- towncrier release notes start -->
+
+## __cylc-8.2.0 (<span actions:bind='release-date'>Released 2023-07-21</span>)__
 
 ### Breaking Changes
 
@@ -26,9 +27,10 @@ Before trying to reload the workflow definition, the scheduler will
 now wait for preparing tasks to submit, and pause the workflow.
 After successful reload the scheduler will unpause the workflow.
 
--[#5605](https://github.com/cylc/cylc-flow/pull/5605) - A shorthand for defining
--a list of strings - Before: `cylc command -s "X=['a', 'bc', 'd']"` - After:
--`cylc command -z X=a,bc,d`.
+[#5605](https://github.com/cylc/cylc-flow/pull/5605) - Added `-z` shorthand
+option for defining a list of strings:
+- Before: `cylc command -s "X=['a', 'bc', 'd']"`
+- After: `cylc command -z X=a,bc,d`.
 
 [#5537](https://github.com/cylc/cylc-flow/pull/5537) - Allow parameters
 in family names to be split, e.g. `<foo>FAM<bar>`.

CONTRIBUTING.md

@@ -36,6 +36,11 @@ Feel free to ask questions on the issue or
 [developers chat](https://matrix.to/#/#cylc-general:matrix.org) if unsure about
 anything.
 
+We use [towncrier](https://towncrier.readthedocs.io/en/stable/index.html) for
+generating the changelog. Changelog entries are added by running
+```
+towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
+```
 
 ## Code Contributors
 

changes.d/5631.fix.md

@@ -0,0 +1 @@
+Fix bug in remote clean for workflows that generated `flow.cylc` files at runtime.

changes.d/changelog-template.jinja

@@ -0,0 +1,13 @@
+{% if sections[""] %}
+{% for category, val in definitions.items() if category in sections[""] %}
+### {{ definitions[category]['name'] }}
+
+{% for text, pulls in sections[""][category].items() %}
+{{ pulls|join(', ') }} - {{ text }}
+
+{% endfor %}
+{% endfor %}
+{% else %}
+No significant changes.
+
+{% endif %}

cylc/flow/__init__.py

@@ -53,7 +53,7 @@ def environ_init():
 
 environ_init()
 
-__version__ = '8.2.0.dev'
+__version__ = '8.2.1.dev'
 
 
 def iter_entry_points(entry_point_name):

cylc/flow/clean.py

@@ -341,7 +341,7 @@ def remote_clean(
     rm_dirs: Optional[List[str]] = None,
     timeout: str = '120'
 ) -> None:
-    """Run subprocesses to clean workflows on remote install targets
+    """Run subprocesses to clean a workflow on its remote install targets
     (skip localhost), given a set of platform names to look up.
 
     Args:
@@ -446,7 +446,7 @@ def _remote_clean_cmd(
         f"Cleaning {id_} on install target: {platform['install target']} "
         f"(using platform: {platform['name']})"
     )
-    cmd = ['clean', '--local-only', id_]
+    cmd = ['clean', '--local-only', '--no-scan', id_]
     if rm_dirs is not None:
         for item in rm_dirs:
             cmd.extend(['--rm', item])

cylc/flow/command_polling.py

@@ -28,17 +28,14 @@ def add_to_cmd_options(cls, parser, d_interval=60, d_max_polls=10):
         """Add command line options for commands that can do polling"""
         parser.add_option(
             "--max-polls",
-            help="Maximum number of polls (default " + str(d_max_polls) + ").",
+            help=r"Maximum number of polls (default: %default).",
             metavar="INT",
             action="store",
             dest="max_polls",
             default=d_max_polls)
         parser.add_option(
             "--interval",
-            help=(
-                "Polling interval in seconds (default " + str(d_interval) +
-                ")."
-            ),
+            help=r"Polling interval in seconds (default: %default).",
             metavar="SECS",
             action="store",
             dest="interval",

cylc/flow/scheduler_cli.py

@@ -244,7 +244,7 @@
     ),
     OptionSettings(
         ["--format"],
-        help="The format of the output: 'plain'=human readable, 'json",
+        help="The format of the output: 'plain'=human readable, 'json'",
         choices=('plain', 'json'),
         default="plain",
         dest='format',