diff --git a/.github/workflows/1_create_release_pr.yml b/.github/workflows/1_create_release_pr.yml index 9f537369f4d..f94b8d60873 100644 --- a/.github/workflows/1_create_release_pr.yml +++ b/.github/workflows/1_create_release_pr.yml @@ -43,15 +43,14 @@ jobs: init-file: 'cylc/flow/__init__.py' pypi-package-name: 'cylc-flow' - - name: Update "released on" date in changelog - continue-on-error: true - uses: cylc/release-actions/stage-1/update-changelog-release-date@v1 - with: - changelog-file: 'CHANGES.md' - - name: Test build uses: cylc/release-actions/build-python-package@v1 + - name: Generate changelog + run: | + python3 -m pip install -q towncrier + towncrier build --yes + - name: Create pull request uses: cylc/release-actions/stage-1/create-release-pr@v1 with: diff --git a/.github/workflows/branch_sync.yml b/.github/workflows/branch_sync.yml index c4334e465a3..5e783ef3ad5 100644 --- a/.github/workflows/branch_sync.yml +++ b/.github/workflows/branch_sync.yml @@ -4,9 +4,11 @@ on: push: branches: - '8.*.x' + schedule: + - cron: '33 04 * * 1-5' # 04:33 UTC Mon-Fri workflow_dispatch: inputs: - branch: + head_branch: description: Branch to merge into master required: true @@ -15,26 +17,53 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 env: - BRANCH: ${{ inputs.branch || github.ref_name }} + HEAD_BRANCH: ${{ inputs.head_branch || github.ref_name }} + STATUS_JSON: https://raw.githubusercontent.com/cylc/cylc-admin/master/docs/status/branches.json steps: - name: Check branch name shell: python run: | import os + import json import sys + from urllib.request import urlopen + + if os.environ['GITHUB_EVENT_NAME'] == 'schedule': + # Get branch from status page + meta = json.loads( + urlopen(os.environ['STATUS_JSON']).read() + )['meta_releases'] + version = min(meta) + branch = meta[version][os.environ['GITHUB_REPOSITORY']] + else: + branch = os.environ['HEAD_BRANCH'].strip() - branch = os.environ['BRANCH'].strip() - if not branch: - sys.exit("::error::Branch name cannot be empty") - if branch.endswith('deconflict'): - sys.exit("::error::Do not run this workflow for already-created deconflict branches") + if branch.endswith('-sync'): + sys.exit("::error::Do not run this workflow for already-created sync branches") with open(os.environ['GITHUB_ENV'], 'a') as F: - print(f'BRANCH={branch}', file=F) - print(f'DECONFLICT_BRANCH={branch}-deconflict', file=F) + print(f'HEAD_BRANCH={branch}', file=F) + print(f'SYNC_BRANCH={branch}-sync', file=F) + + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: master + + - name: Configure git + uses: cylc/release-actions/configure-git@v1 + + - name: Attempt fast-forward + id: ff + continue-on-error: true + run: | + git merge "origin/${HEAD_BRANCH}" --ff-only + git push origin master - name: Check for existing PR id: check-pr + if: steps.ff.outcome == 'failure' shell: python env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -44,7 +73,7 @@ jobs: import subprocess import sys - for env_var in ('BRANCH', 'DECONFLICT_BRANCH'): + for env_var in ('HEAD_BRANCH', 'SYNC_BRANCH'): branch = os.environ[env_var] cmd = f'gh pr list -B master -H {branch} -s open --json url -R ${{ github.repository }}' ret = subprocess.run( @@ -55,66 +84,64 @@ jobs: print(f"::error::{ret.stderr}") if ret.returncode: sys.exit(ret.returncode) - if json.loads(ret.stdout): - print(f"::notice::Found existing PR for {branch}") + results: list = json.loads(ret.stdout) + if results: + print(f"::notice::Found existing PR for {branch} - {results[0]['url']}") sys.exit(0) print("No open PRs found") with open(os.environ['GITHUB_OUTPUT'], 'a') as f: print('continue=true', file=f) - - name: Checkout - if: steps.check-pr.outputs.continue - uses: actions/checkout@v3 - with: - fetch-depth: 0 - ref: master - - - name: Configure git - if: steps.check-pr.outputs.continue - uses: cylc/release-actions/configure-git@v1 - - name: Attempt merge id: merge if: steps.check-pr.outputs.continue continue-on-error: true - run: git merge "origin/${BRANCH}" + run: git merge "origin/${HEAD_BRANCH}" + + - name: Abort merge + if: steps.merge.outcome == 'failure' + run: git merge --abort - name: Diff id: diff if: steps.merge.outcome == 'success' run: | if [[ "$(git rev-parse HEAD)" == "$(git rev-parse origin/master)" ]]; then - echo "::notice::master is up to date with $BRANCH" + echo "::notice::master is up to date with $HEAD_BRANCH" exit 0 fi if git diff HEAD^ --exit-code --stat; then - echo "::notice::No diff between master and $BRANCH" + echo "::notice::No diff between master and $HEAD_BRANCH" exit 0 fi echo "continue=true" >> $GITHUB_OUTPUT - - name: Create deconflict branch - if: steps.merge.outcome == 'failure' + - name: Push sync branch + id: push + if: steps.merge.outcome == 'failure' || steps.diff.outputs.continue run: | - git merge --abort - git checkout -b "$DECONFLICT_BRANCH" "origin/${BRANCH}" - git push origin "$DECONFLICT_BRANCH" - echo "BRANCH=${DECONFLICT_BRANCH}" >> $GITHUB_ENV + git checkout -b "$SYNC_BRANCH" "origin/${HEAD_BRANCH}" + git push origin "$SYNC_BRANCH" + echo "continue=true" >> $GITHUB_OUTPUT - name: Open PR - if: steps.merge.outcome == 'failure' || steps.diff.outputs.continue + if: steps.push.outputs.continue env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BODY: | - Please do a **normal merge**, not squash merge + Please do a **normal merge**, not squash merge. + Please fix conflicts if necessary. --- Triggered by `${{ github.event_name }}` run: | - gh pr create --head "$BRANCH" \ - --title "🤖 Merge ${BRANCH} into master" \ + url="$( + gh pr create --head "$SYNC_BRANCH" \ + --title "🤖 Merge ${SYNC_BRANCH} into master" \ --body "$BODY" + )" + echo "::notice::PR created at ${url}" - gh pr edit "$BRANCH" --add-label "sync" || true + gh pr edit "$SYNC_BRANCH" --add-label "sync" || true diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml index c4fc5c3308c..6cc1ebb6c14 100644 --- a/.github/workflows/test_fast.yml +++ b/.github/workflows/test_fast.yml @@ -48,6 +48,10 @@ jobs: - name: Configure git # Needed by the odd test uses: cylc/release-actions/configure-git@v1 + - name: Check changelog + if: startsWith(matrix.os, 'ubuntu') + run: towncrier build --draft + - name: Style if: startsWith(matrix.os, 'ubuntu') run: | diff --git a/CHANGES.md b/CHANGES.md index ea903b88676..483023c0dd4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,13 +4,14 @@ List of notable changes, for a complete list of changes see the [closed milestones](https://github.com/cylc/cylc-flow/milestones?state=closed) for each release. - + -## __cylc-8.2.0 (Upcoming)__ + + +## __cylc-8.2.0 (Released 2023-07-21)__ ### Breaking Changes @@ -26,9 +27,10 @@ Before trying to reload the workflow definition, the scheduler will now wait for preparing tasks to submit, and pause the workflow. After successful reload the scheduler will unpause the workflow. --[#5605](https://github.com/cylc/cylc-flow/pull/5605) - A shorthand for defining --a list of strings - Before: `cylc command -s "X=['a', 'bc', 'd']"` - After: --`cylc command -z X=a,bc,d`. +[#5605](https://github.com/cylc/cylc-flow/pull/5605) - Added `-z` shorthand +option for defining a list of strings: +- Before: `cylc command -s "X=['a', 'bc', 'd']"` +- After: `cylc command -z X=a,bc,d`. [#5537](https://github.com/cylc/cylc-flow/pull/5537) - Allow parameters in family names to be split, e.g. `FAM`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 72f63d67d03..6fbc4283968 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,11 @@ Feel free to ask questions on the issue or [developers chat](https://matrix.to/#/#cylc-general:matrix.org) if unsure about anything. +We use [towncrier](https://towncrier.readthedocs.io/en/stable/index.html) for +generating the changelog. Changelog entries are added by running +``` +towncrier create ..md --content "Short description" +``` ## Code Contributors diff --git a/changes.d/5631.fix.md b/changes.d/5631.fix.md new file mode 100644 index 00000000000..a3b4923bd02 --- /dev/null +++ b/changes.d/5631.fix.md @@ -0,0 +1 @@ +Fix bug in remote clean for workflows that generated `flow.cylc` files at runtime. diff --git a/changes.d/changelog-template.jinja b/changes.d/changelog-template.jinja new file mode 100644 index 00000000000..9a96512694c --- /dev/null +++ b/changes.d/changelog-template.jinja @@ -0,0 +1,13 @@ +{% if sections[""] %} +{% for category, val in definitions.items() if category in sections[""] %} +### {{ definitions[category]['name'] }} + +{% for text, pulls in sections[""][category].items() %} +{{ pulls|join(', ') }} - {{ text }} + +{% endfor %} +{% endfor %} +{% else %} +No significant changes. + +{% endif %} diff --git a/cylc/flow/__init__.py b/cylc/flow/__init__.py index f74e9a57cfa..ac1743a704f 100644 --- a/cylc/flow/__init__.py +++ b/cylc/flow/__init__.py @@ -53,7 +53,7 @@ def environ_init(): environ_init() -__version__ = '8.2.0.dev' +__version__ = '8.2.1.dev' def iter_entry_points(entry_point_name): diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py index 992fdd24839..81259e12a25 100644 --- a/cylc/flow/clean.py +++ b/cylc/flow/clean.py @@ -341,7 +341,7 @@ def remote_clean( rm_dirs: Optional[List[str]] = None, timeout: str = '120' ) -> None: - """Run subprocesses to clean workflows on remote install targets + """Run subprocesses to clean a workflow on its remote install targets (skip localhost), given a set of platform names to look up. Args: @@ -446,7 +446,7 @@ def _remote_clean_cmd( f"Cleaning {id_} on install target: {platform['install target']} " f"(using platform: {platform['name']})" ) - cmd = ['clean', '--local-only', id_] + cmd = ['clean', '--local-only', '--no-scan', id_] if rm_dirs is not None: for item in rm_dirs: cmd.extend(['--rm', item]) diff --git a/cylc/flow/command_polling.py b/cylc/flow/command_polling.py index b36709e24aa..dcf186edbd9 100644 --- a/cylc/flow/command_polling.py +++ b/cylc/flow/command_polling.py @@ -28,17 +28,14 @@ def add_to_cmd_options(cls, parser, d_interval=60, d_max_polls=10): """Add command line options for commands that can do polling""" parser.add_option( "--max-polls", - help="Maximum number of polls (default " + str(d_max_polls) + ").", + help=r"Maximum number of polls (default: %default).", metavar="INT", action="store", dest="max_polls", default=d_max_polls) parser.add_option( "--interval", - help=( - "Polling interval in seconds (default " + str(d_interval) + - ")." - ), + help=r"Polling interval in seconds (default: %default).", metavar="SECS", action="store", dest="interval", diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py index b38b797b878..3ae1abddadd 100644 --- a/cylc/flow/scheduler_cli.py +++ b/cylc/flow/scheduler_cli.py @@ -244,7 +244,7 @@ ), OptionSettings( ["--format"], - help="The format of the output: 'plain'=human readable, 'json", + help="The format of the output: 'plain'=human readable, 'json'", choices=('plain', 'json'), default="plain", dest='format', diff --git a/cylc/flow/scripts/clean.py b/cylc/flow/scripts/clean.py index c2d2ee3032a..94fe92b6dab 100644 --- a/cylc/flow/scripts/clean.py +++ b/cylc/flow/scripts/clean.py @@ -60,6 +60,7 @@ """ import asyncio +from optparse import SUPPRESS_HELP import sys from typing import TYPE_CHECKING, Iterable, List, Tuple @@ -120,10 +121,17 @@ def get_option_parser(): parser.add_option( '--timeout', help=("The number of seconds to wait for cleaning to take place on " - "remote hosts before cancelling."), + r"remote hosts before cancelling. Default: %default."), action='store', default='120', dest='remote_timeout' ) + parser.add_option( + '--no-scan', + help=SUPPRESS_HELP, action='store_true', dest='no_scan' + # Used on remote re-invocation - do not scan for workflows, just + # clean exactly what you were told to clean + ) + return parser @@ -173,28 +181,31 @@ async def scan( async def run(*ids: str, opts: 'Values') -> None: - # parse ids from the CLI - workflows, multi_mode = await parse_ids_async( - *ids, - constraint='workflows', - match_workflows=True, - match_active=False, - infer_latest_runs=False, # don't infer latest runs like other cmds - ) + if opts.no_scan: + workflows: Iterable[str] = ids + else: + # parse ids from the CLI + workflows, multi_mode = await parse_ids_async( + *ids, + constraint='workflows', + match_workflows=True, + match_active=False, + infer_latest_runs=False, # don't infer latest runs like other cmds + ) - # expand partial workflow ids (including run names) - workflows, multi_mode = await scan(workflows, multi_mode) + # expand partial workflow ids (including run names) + workflows, multi_mode = await scan(workflows, multi_mode) - if not workflows: - LOG.warning(f"No workflows matching {', '.join(ids)}") - return + if not workflows: + LOG.warning(f"No workflows matching {', '.join(ids)}") + return - workflows.sort() - if multi_mode and not opts.skip_interactive: - prompt(workflows) # prompt for approval or exit + workflows.sort() + if multi_mode and not opts.skip_interactive: + prompt(workflows) # prompt for approval or exit failed = {} - for workflow in sorted(workflows): + for workflow in workflows: try: init_clean(workflow, opts) except Exception as exc: diff --git a/cylc/flow/scripts/cycle_point.py b/cylc/flow/scripts/cycle_point.py index 11cc9c9d1a1..bbb030ef0ab 100755 --- a/cylc/flow/scripts/cycle_point.py +++ b/cylc/flow/scripts/cycle_point.py @@ -115,8 +115,8 @@ def get_option_parser() -> COP: parser.add_option( "--time-zone", metavar="TEMPLATE", - help="Control the formatting of the result's timezone e.g. " - "(Z, +13:00, -hh", + help="Control the formatting of the result's timezone (e.g. " + "Z, +13:00, -hh)", action="store", default=None, dest="time_zone") parser.add_option( diff --git a/cylc/flow/scripts/ext_trigger.py b/cylc/flow/scripts/ext_trigger.py index d33f4c1a91a..c76bbb86712 100755 --- a/cylc/flow/scripts/ext_trigger.py +++ b/cylc/flow/scripts/ext_trigger.py @@ -89,14 +89,22 @@ def get_option_parser() -> COP: ) parser.add_option( - "--max-tries", help="Maximum number of send attempts " - "(default %s)." % MAX_N_TRIES, metavar="INT", - action="store", default=MAX_N_TRIES, dest="max_n_tries") + "--max-tries", + help=r"Maximum number of send attempts (default: %default).", + metavar="INT", + action="store", + default=MAX_N_TRIES, + dest="max_n_tries" + ) parser.add_option( - "--retry-interval", help="Delay in seconds before retrying " - "(default %s)." % RETRY_INTVL_SECS, metavar="SEC", - action="store", default=RETRY_INTVL_SECS, dest="retry_intvl_secs") + "--retry-interval", + help=r"Delay in seconds before retrying (default: %default).", + metavar="SEC", + action="store", + default=RETRY_INTVL_SECS, + dest="retry_intvl_secs" + ) return parser diff --git a/cylc/flow/scripts/scan.py b/cylc/flow/scripts/scan.py index 66e7e76a86d..e07e4e6cf07 100644 --- a/cylc/flow/scripts/scan.py +++ b/cylc/flow/scripts/scan.py @@ -209,7 +209,7 @@ def get_option_parser() -> COP: parser.add_option( '--format', '-t', help=( - 'Output data and format (default "plain").' + r'Output data and format (default "%default").' ' ("name": list the workflow IDs only)' ' ("plain": name,host:port,PID on one line)' ' ("tree": name,host:port,PID in tree format)' diff --git a/etc/bin/swarm b/etc/bin/swarm index 9233e1de1fb..6e8c814d49e 100755 --- a/etc/bin/swarm +++ b/etc/bin/swarm @@ -204,7 +204,7 @@ append_config () { if prompt "Write \"$LINE\" to \"$LOC\"?"; then if [[ "$POS" == top ]]; then # ... at the top of the file - sed -i "1i$LINE" "$LOC" + echo -e "${LINE}\n$(cat "$LOC")" > "$LOC" elif [[ "$POS" == bottom ]]; then # ... at the bottom of the file echo -e "\n$LINE" >> "$LOC" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..6bbfe0507a1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[tool.towncrier] +directory = "changes.d" +name = "Cylc" +package = "cylc.flow" +filename = "CHANGES.md" +template = "changes.d/changelog-template.jinja" +underlines = ["", "", ""] +title_format = "## __cylc-{version} (Released {project_date})__" +issue_format = "[#{issue}](https://github.com/cylc/cylc-flow/pull/{issue})" + +# These changelog sections will be shown in the defined order: +[[tool.towncrier.type]] +directory = "break" # NB this is just the filename not directory e.g. 123.break.md +name = "⚠ Breaking Changes" +showcontent = true +[[tool.towncrier.type]] +directory = "feat" +name = "🚀 Enhancements" +showcontent = true +[[tool.towncrier.type]] +directory = "fix" +name = "🔧 Fixes" +showcontent = true diff --git a/setup.cfg b/setup.cfg index cb5244a9509..eddfd22834e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -122,6 +122,7 @@ tests = pytest-mock>=3.6.1 pytest>=6 testfixtures>=6.11.0 + towncrier>=23 # Type annotation stubs # http://mypy-lang.blogspot.com/2021/05/the-upcoming-switch-to-modular-typeshed.html types-Jinja2>=0.1.3 diff --git a/tests/functional/cylc-clean/01-remote.t b/tests/functional/cylc-clean/01-remote.t index 64a01caffcb..6bc4a9a40ea 100644 --- a/tests/functional/cylc-clean/01-remote.t +++ b/tests/functional/cylc-clean/01-remote.t @@ -50,6 +50,7 @@ init_workflow "${TEST_NAME_BASE}" << __FLOW__ [runtime] [[root]] platform = ${CYLC_TEST_PLATFORM} + script = touch flow.cylc # testing that remote clean does not scan for workflows __FLOW__ FUNCTIONAL_DIR="${TEST_SOURCE_DIR_BASE%/*}" @@ -94,6 +95,8 @@ ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${CYLC_TEST_REG_BASE} | \`-- cycle -> ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${WORKFLOW_NAME}/share/cycle \`-- work \`-- 1 + \`-- santa + \`-- flow.cylc ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE} \`-- ${FUNCTIONAL_DIR} \`-- cylc-clean @@ -135,4 +138,3 @@ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE} __TREE__ purge -exit diff --git a/tests/unit/test_clean.py b/tests/unit/test_clean.py index 61c8964fd3c..b7769cfb54d 100644 --- a/tests/unit/test_clean.py +++ b/tests/unit/test_clean.py @@ -992,9 +992,11 @@ def test_remote_clean_cmd( monkeymock('cylc.flow.clean.Popen') cylc_clean._remote_clean_cmd(id_, platform, rm_dirs, timeout='dunno') - args, kwargs = mock_construct_ssh_cmd.call_args + args, _kwargs = mock_construct_ssh_cmd.call_args constructed_cmd = args[0] - assert constructed_cmd == ['clean', '--local-only', id_, *expected_args] + assert constructed_cmd == [ + 'clean', '--local-only', '--no-scan', id_, *expected_args + ] def test_clean_top_level(tmp_run_dir: Callable):