diff --git a/.github/workflows/1_create_release_pr.yml b/.github/workflows/1_create_release_pr.yml index 6d792304196..1595004b940 100644 --- a/.github/workflows/1_create_release_pr.yml +++ b/.github/workflows/1_create_release_pr.yml @@ -47,9 +47,7 @@ jobs: uses: cylc/release-actions/build-python-package@v1 - name: Generate changelog - run: | - python3 -m pip install -q towncrier - towncrier build --yes + uses: MetRonnie/release-actions/stage-1/towncrier-build@changelog - name: Create pull request uses: cylc/release-actions/stage-1/create-release-pr@v1 diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml index 02ea52d678c..d98688366ed 100644 --- a/.github/workflows/2_auto_publish_release.yml +++ b/.github/workflows/2_auto_publish_release.yml @@ -45,6 +45,14 @@ jobs: # # Can try using this for testing: # repository_url: https://test.pypi.org/legacy/ + - name: Write release notes + id: release-notes + uses: MetRonnie/release-actions/stage-2/write-release-notes@changelog + with: + footer: | + Cylc 8 can be installed via pypi or Conda - you don't need to download this release directly. + See the [installation](https://cylc.github.io/cylc-doc/latest/html/installation.html) section of the documentation. + - name: Publish GitHub release id: create-release uses: cylc/release-actions/create-release@v1 @@ -55,12 +63,7 @@ jobs: tag_name: ${{ env.VERSION }} release_name: cylc-flow-${{ env.VERSION }} prerelease: ${{ env.PRERELEASE }} - body: | - See [${{ env.CHANGELOG_FILE }}](https://github.com/${{ github.repository }}/blob/master/${{ env.CHANGELOG_FILE }}) for detail. - - Cylc 8 can be installed via pypi or Conda - you don't need to download this release directly. - See the [installation](https://cylc.github.io/cylc-doc/latest/html/installation.html) section of the documentation. - # TODO: Get topmost changelog section somehow and use that as the body? + body_path: ${{ steps.release-notes.outputs.filepath }} - name: Comment on the release PR with the results & next steps if: always() diff --git a/.github/workflows/branch_sync.yml b/.github/workflows/branch_sync.yml index d0e5be512e5..92475094217 100644 --- a/.github/workflows/branch_sync.yml +++ b/.github/workflows/branch_sync.yml @@ -14,109 +14,7 @@ on: jobs: sync: - runs-on: ubuntu-latest - timeout-minutes: 5 - env: - BASE_BRANCH: master - HEAD_BRANCH: ${{ inputs.head_branch || github.ref_name }} - STATUS_JSON: https://raw.githubusercontent.com/cylc/cylc-admin/master/docs/status/branches.json - FORCE_COLOR: 2 - steps: - - name: Check branch name - shell: python - run: | - import os - import json - import sys - from urllib.request import urlopen - - if os.environ['GITHUB_EVENT_NAME'] == 'schedule': - # Get branch from status page - meta = json.loads( - urlopen(os.environ['STATUS_JSON']).read() - )['meta_releases'] - version = min(meta) - branch = meta[version][os.environ['GITHUB_REPOSITORY']] - else: - branch = os.environ['HEAD_BRANCH'].strip() - - if branch.endswith('-sync'): - sys.exit("::error::Do not run this workflow for already-created sync branches") - - with open(os.environ['GITHUB_ENV'], 'a') as F: - print(f'HEAD_BRANCH={branch}', file=F) - print(f'SYNC_BRANCH={branch}-sync', file=F) - - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: master - - - name: Configure git - uses: cylc/release-actions/configure-git@v1 - - - name: Checkout sync branch if it exists - continue-on-error: true - run: | - git switch -c "$SYNC_BRANCH" "origin/${SYNC_BRANCH}" - echo "BASE_BRANCH=$SYNC_BRANCH" >> "$GITHUB_ENV" - - - name: Attempt fast-forward - id: ff - run: | - if git merge "origin/${HEAD_BRANCH}" --ff-only; then - if [[ "$(git rev-parse HEAD)" == "$(git rev-parse "origin/${BASE_BRANCH}")" ]]; then - echo "::notice::$BASE_BRANCH is up to date with $HEAD_BRANCH" - exit 0 - fi - git push origin "$BASE_BRANCH" - elif [[ "$BASE_BRANCH" == "$SYNC_BRANCH" ]]; then - echo "::notice::Cannot fast-forward $BASE_BRANCH to $HEAD_BRANCH; merge existing PR first" - else - echo "continue=true" >> "$GITHUB_OUTPUT" - fi - - - name: Attempt merge into master - id: merge - if: steps.ff.outputs.continue - run: | - git switch master - if git merge "origin/${HEAD_BRANCH}"; then - if git diff HEAD^ --exit-code --stat; then - echo "::notice::No diff between master and $HEAD_BRANCH" - exit 0 - fi - else - git merge --abort - fi - echo "continue=true" >> $GITHUB_OUTPUT - - - name: Push sync branch - id: push - if: steps.merge.outputs.continue - run: | - git switch -c "$SYNC_BRANCH" "origin/${HEAD_BRANCH}" - git push origin "$SYNC_BRANCH" - echo "continue=true" >> $GITHUB_OUTPUT - - - name: Open PR - if: steps.push.outputs.continue - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BODY: | - Please do a **normal merge**, not squash merge. - Please fix conflicts if necessary. - - --- - - Triggered by `${{ github.event_name }}` - run: | - url="$( - gh pr create --head "$SYNC_BRANCH" \ - --title "🤖 Merge ${SYNC_BRANCH} into master" \ - --body "$BODY" - )" - echo "::notice::PR created at ${url}" - - gh pr edit "$SYNC_BRANCH" --add-label "sync" || true + uses: cylc/release-actions/.github/workflows/branch-sync.yml@v1 + with: + head_branch: ${{ inputs.head_branch }} + secrets: inherit diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml index d0bff50326d..2f303e36ee7 100644 --- a/.github/workflows/test_fast.yml +++ b/.github/workflows/test_fast.yml @@ -17,10 +17,10 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: - fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs + fail-fast: false # don't stop on first failure matrix: os: ['ubuntu-latest'] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3'] include: - os: 'macos-latest' python-version: '3.7' @@ -39,7 +39,7 @@ jobs: if: startsWith(matrix.os, 'ubuntu') run: | sudo apt-get update - sudo apt-get install -y shellcheck sqlite3 + sudo apt-get install -y sqlite3 - name: Install run: | @@ -48,37 +48,10 @@ jobs: - name: Configure git # Needed by the odd test uses: cylc/release-actions/configure-git@v1 - - name: Check changelog - if: startsWith(matrix.os, 'ubuntu') - run: towncrier build --draft - - - name: Style - if: startsWith(matrix.os, 'ubuntu') - run: | - flake8 - etc/bin/shellchecker - - # note: exclude python 3.10+ from mypy checks as these produce false - # positives in installed libraries for python 3.7 - - name: Typing - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python-version, 3.9) - run: mypy - - - name: Doctests - timeout-minutes: 4 - run: | - pytest cylc/flow - - name: Unit Tests - timeout-minutes: 4 - run: | - pytest tests/unit - - - name: Bandit - if: ${{ matrix.python-version == '3.7' }} - # https://github.com/PyCQA/bandit/issues/658 + timeout-minutes: 5 run: | - bandit -r --ini .bandit cylc/flow + pytest cylc/flow tests/unit - name: Integration Tests timeout-minutes: 6 @@ -104,9 +77,47 @@ jobs: path: coverage.xml retention-days: 7 + lint: + runs-on: 'ubuntu-latest' + timeout-minutes: 10 + steps: + - name: Apt-Get Install + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + + - name: Checkout + uses: actions/checkout@v4 + + # note: exclude python 3.10+ from mypy checks as these produce false + # positives in installed libraries for python 3.7 + - name: Configure Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install + run: | + pip install -e ."[tests]" + + - name: Flake8 + run: flake8 + + - name: Bandit + run: | + bandit -r --ini .bandit cylc/flow + + - name: Shellchecker + run: etc/bin/shellchecker + + - name: MyPy + run: mypy + + - name: Towncrier + run: towncrier build --draft + - name: Linkcheck - if: startsWith(matrix.python-version, '3.10') - run: pytest -m linkcheck --dist=load tests/unit + run: pytest -m linkcheck --dist=load --color=yes -n 10 tests/unit/test_links.py codecov: needs: test diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml index 75dac06a964..d3a6a38dad3 100644 --- a/.github/workflows/test_functional.yml +++ b/.github/workflows/test_functional.yml @@ -46,9 +46,9 @@ jobs: # NOTE: includes must define ALL of the matrix values include: # latest python - - name: 'py-3.11' + - name: 'py-3-latest' os: 'ubuntu-latest' - python-version: '3.11' + python-version: '3' test-base: 'tests/f' chunk: '1/4' platform: '_local_background*' @@ -108,7 +108,7 @@ jobs: run: | # install system deps brew update - brew install bash coreutils gnu-sed + brew install bash coreutils gnu-sed grep # add GNU coreutils and sed to the user PATH # (see instructions in brew install output) @@ -118,6 +118,9 @@ jobs: echo \ "/usr/local/opt/gnu-sed/libexec/gnubin" \ >> "${GITHUB_PATH}" + echo \ + "/usr/local/opt/grep/libexec/gnubin" \ + >> "${GITHUB_PATH}" # add coreutils to the bashrc too (for jobs) cat >> "${HOME}/.bashrc" <<__HERE__ diff --git a/CHANGES.md b/CHANGES.md index b34ed2b4d73..c31789a8019 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,10 +19,18 @@ $ towncrier create ..md --content "Short description" ### 🔧 Fixes +WHATEVER +## __cylc-8.2.3 (Released 2023-11-02)__ + +### 🔧 Fixes + +[#5660](https://github.com/cylc/cylc-flow/pull/5660) - Re-worked graph n-window algorithm for better efficiency. [#5753](https://github.com/cylc/cylc-flow/pull/5753) - Fixed bug where execution time limit polling intervals could end up incorrectly applied [#5776](https://github.com/cylc/cylc-flow/pull/5776) - Ensure that submit-failed tasks are marked as incomplete (so remain visible) when running in back-compat mode. +[#5791](https://github.com/cylc/cylc-flow/pull/5791) - fix a bug where if multiple clock triggers are set for a task only one was being satisfied. + ## __cylc-8.2.2 (Released 2023-10-05)__ ### 🚀 Enhancements diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6d27175ff9..a1bf42e6215 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ requests_). - Prasanna Challuri - David Matthews - Tim Whitcomb - - (Scott Wales) + - Scott Wales - Tomek Trzeciak - Thomas Coleman - Bruno Kinoshita diff --git a/changes.d/5709.feat.md b/changes.d/5709.feat.md new file mode 100644 index 00000000000..11aeabcf81d --- /dev/null +++ b/changes.d/5709.feat.md @@ -0,0 +1 @@ +Forward arbitrary environment variables over SSH connections diff --git a/changes.d/5727.break.md b/changes.d/5727.break.md new file mode 100644 index 00000000000..06cb196216d --- /dev/null +++ b/changes.d/5727.break.md @@ -0,0 +1 @@ +Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value. If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`. \ No newline at end of file diff --git a/changes.d/5731.feat.md b/changes.d/5731.feat.md new file mode 100644 index 00000000000..b0c28a01ac1 --- /dev/null +++ b/changes.d/5731.feat.md @@ -0,0 +1 @@ +Major upgrade to `cylc tui` which now supports larger workflows and can browse installed workflows. diff --git a/changes.d/5772.feat.md b/changes.d/5772.feat.md new file mode 100644 index 00000000000..9d47d528622 --- /dev/null +++ b/changes.d/5772.feat.md @@ -0,0 +1 @@ +Add a check for indentation being 4N spaces. \ No newline at end of file diff --git a/changes.d/5794.break.md b/changes.d/5794.break.md new file mode 100644 index 00000000000..53c5315b013 --- /dev/null +++ b/changes.d/5794.break.md @@ -0,0 +1 @@ +Remove `cylc report-timings` from automatic installation with `pip install cylc-flow[all]`. If you now wish to install it use `pip install cylc-flow[report-timings]`. `cylc report-timings` is incompatible with Python 3.12. \ No newline at end of file diff --git a/changes.d/5801.fix.md b/changes.d/5801.fix.md new file mode 100644 index 00000000000..e7fd0584090 --- /dev/null +++ b/changes.d/5801.fix.md @@ -0,0 +1 @@ +Fix traceback when using parentheses on right hand side of graph trigger. diff --git a/changes.d/5803.feat.md b/changes.d/5803.feat.md new file mode 100644 index 00000000000..a4bc0f1b898 --- /dev/null +++ b/changes.d/5803.feat.md @@ -0,0 +1 @@ +Updated 'reinstall' functionality to support multiple workflows \ No newline at end of file diff --git a/changes.d/5821.fix.md b/changes.d/5821.fix.md new file mode 100644 index 00000000000..0c6c8b7918d --- /dev/null +++ b/changes.d/5821.fix.md @@ -0,0 +1 @@ +Fixed issue where large uncommitted changes could cause `cylc install` to hang. diff --git a/changes.d/5836.break.md b/changes.d/5836.break.md new file mode 100644 index 00000000000..8c14b101f63 --- /dev/null +++ b/changes.d/5836.break.md @@ -0,0 +1 @@ +Removed the 'CYLC_TASK_DEPENDENCIES' environment variable \ No newline at end of file diff --git a/changes.d/5838.feat.md b/changes.d/5838.feat.md new file mode 100644 index 00000000000..8e9919d3a0f --- /dev/null +++ b/changes.d/5838.feat.md @@ -0,0 +1 @@ +`cylc lint`: added rule to check for `rose date` usage (should be replaced with `isodatetime`). diff --git a/changes.d/5841.fix.md b/changes.d/5841.fix.md new file mode 100644 index 00000000000..9e11386b8d0 --- /dev/null +++ b/changes.d/5841.fix.md @@ -0,0 +1 @@ +Improve handling of S011 to not warn if the # is '#$' (e.g. shell base arithmetic) \ No newline at end of file diff --git a/conda-environment.yml b/conda-environment.yml index 11dd58f7521..93f5cda68fc 100644 --- a/conda-environment.yml +++ b/conda-environment.yml @@ -9,14 +9,14 @@ dependencies: - graphviz # for static graphing # Note: can't pin jinja2 any higher than this until we give up on Cylc 7 back-compat - jinja2 >=3.0,<3.1 - - metomi-isodatetime >=1!3.0.0, <1!3.1.0 + - metomi-isodatetime >=1!3.0.0, <1!3.2.0 + - packaging # Constrain protobuf version for compatible Scheduler-UIS comms across hosts - - protobuf >=4.21.2,<4.22.0 + - protobuf >=4.24.4,<4.25.0 - psutil >=5.6.0 - python - pyzmq >=22 - - setuptools >=49,!=67.* - - importlib_metadata # [py<3.8] + - importlib_metadata >=5.0 # [py<3.12] - urwid >=2,<3 - tomli >=2 # [py<3.11] diff --git a/cylc/flow/__init__.py b/cylc/flow/__init__.py index 5acb550aedc..b3a7f0487cf 100644 --- a/cylc/flow/__init__.py +++ b/cylc/flow/__init__.py @@ -58,11 +58,19 @@ def environ_init(): def iter_entry_points(entry_point_name): """Iterate over Cylc entry points.""" - import pkg_resources + import sys + if sys.version_info[:2] > (3, 11): + from importlib.metadata import entry_points + else: + # BACK COMPAT: importlib_metadata + # importlib.metadata was added in Python 3.8. The required interfaces + # were completed by 3.12. For lower versions we must use the + # importlib_metadata backport. + # FROM: Python 3.7 + # TO: Python: 3.12 + from importlib_metadata import entry_points yield from ( entry_point - for entry_point in pkg_resources.iter_entry_points(entry_point_name) - # Filter out the cylc namespace as it should be empty. - # All cylc packages should take the form cylc- - if entry_point.dist.key != 'cylc' + # for entry_point in entry_points()[entry_point_name] + for entry_point in entry_points().select(group=entry_point_name) ) diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 86c639af113..2d39ad74829 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -22,7 +22,7 @@ from typing import List, Optional, Tuple, Any, Union from contextlib import suppress -from pkg_resources import parse_version +from packaging.version import Version from cylc.flow import LOG from cylc.flow import __version__ as CYLC_VERSION @@ -1222,6 +1222,9 @@ def default_for( {PLATFORM_REPLACES.format("[job]batch system")} ''') + replaces = PLATFORM_REPLACES.format( + "[job]batch submit command template" + ) Conf('job runner command template', VDR.V_STRING, desc=f''' Set the command used by the chosen job runner. @@ -1230,9 +1233,7 @@ def default_for( .. versionadded:: 8.0.0 - {PLATFORM_REPLACES.format( - "[job]batch submit command template" - )} + {replaces} ''') Conf('shell', VDR.V_STRING, '/bin/bash', desc=''' @@ -1465,6 +1466,8 @@ def default_for( {REPLACES}``global.rc[hosts][]retrieve job logs command``. ''') + replaces = PLATFORM_REPLACES.format( + "[remote]retrieve job logs max size") Conf('retrieve job logs max size', VDR.V_STRING, desc=f''' {LOG_RETR_SETTINGS['retrieve job logs max size']} @@ -1472,9 +1475,10 @@ def default_for( {REPLACES}``global.rc[hosts][]retrieve job logs max size``. - {PLATFORM_REPLACES.format( - "[remote]retrieve job logs max size")} + {replaces} ''') + replaces = PLATFORM_REPLACES.format( + "[remote]retrieve job logs retry delays") Conf('retrieve job logs retry delays', VDR.V_INTERVAL_LIST, desc=f''' {LOG_RETR_SETTINGS['retrieve job logs retry delays']} @@ -1483,8 +1487,7 @@ def default_for( {REPLACES}``global.rc[hosts][]retrieve job logs retry delays``. - {PLATFORM_REPLACES.format( - "[remote]retrieve job logs retry delays")} + {replaces} ''') Conf('tail command template', VDR.V_STRING, 'tail -n +1 --follow=name %(filename)s', @@ -1652,6 +1655,14 @@ def default_for( .. versionadded:: 8.0.0 ''') + Conf('ssh forward environment variables', VDR.V_STRING_LIST, '', + desc=''' + A list containing the names of the environment variables to + forward with SSH connections to the workflow host from + the host running 'cylc play' + + .. versionadded:: 8.3.0 + ''') with Conf('selection', desc=''' How to select a host from the list of platform hosts. @@ -1858,8 +1869,7 @@ def get_version_hierarchy(version: str) -> List[str]: ['', '8', '8.0', '8.0.1', '8.0.1a2', '8.0.1a2.dev'] """ - smart_ver: Any = parse_version(version) - # (No type anno. yet for Version in pkg_resources.extern.packaging.version) + smart_ver = Version(version) base = [str(i) for i in smart_ver.release] hierarchy = [''] hierarchy += ['.'.join(base[:i]) for i in range(1, len(base) + 1)] diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py index a6d16bbe76d..d75013fd621 100644 --- a/cylc/flow/cfgspec/workflow.py +++ b/cylc/flow/cfgspec/workflow.py @@ -401,7 +401,7 @@ def get_script_common_text(this: str, example: Optional[str] = None): # differentiate between not set vs set to empty default = None elif item.endswith("handlers"): - desc = desc + '\n\n' + dedent(rf''' + desc = desc + '\n\n' + dedent(f''' Examples: .. code-block:: cylc @@ -413,9 +413,9 @@ def get_script_common_text(this: str, example: Optional[str] = None): {item} = echo %(workflow)s # configure multiple event handlers - {item} = \ - 'echo %(workflow)s, %(event)s', \ - 'my_exe %(event)s %(message)s' \ + {item} = \\ + 'echo %(workflow)s, %(event)s', \\ + 'my_exe %(event)s %(message)s' \\ 'curl -X PUT -d event=%(event)s host:port' ''') elif item.startswith("abort on"): @@ -1260,10 +1260,17 @@ def get_script_common_text(this: str, example: Optional[str] = None): - ``all`` - all instance of the task will fail - ``2017-08-12T06, 2017-08-12T18`` - these instances of the task will fail + + If you set :cylc:conf:`[..][..]execution retry delays` + the second attempt will succeed unless you set + :cylc:conf:`[..]fail try 1 only = False`. ''') Conf('fail try 1 only', VDR.V_BOOLEAN, True, desc=''' If ``True`` only the first run of the task instance will fail, otherwise retries will fail too. + + Task instances must be set to fail by + :cylc:conf:`[..]fail cycle points`. ''') Conf('disable task event handlers', VDR.V_BOOLEAN, True, desc=''' @@ -1849,7 +1856,7 @@ def upg(cfg, descr): ['scheduling', 'max active cycle points'], ['scheduling', 'runahead limit'], cvtr=converter( - lambda x: f'P{int(x)-1}' if x != '' else '', + lambda x: f'P{int(x) - 1}' if x != '' else '', '"{old}" -> "{new}"' ), silent=cylc.flow.flags.cylc7_back_compat, diff --git a/cylc/flow/config.py b/cylc/flow/config.py index 0096d924c8b..d80456266bf 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -79,8 +79,8 @@ get_cylc_run_dir, is_relative_to, ) -from cylc.flow.platforms import FORBIDDEN_WITH_PLATFORM from cylc.flow.print_tree import print_tree +from cylc.flow.simulation import configure_sim_modes from cylc.flow.subprocctx import SubFuncContext from cylc.flow.task_events_mgr import ( EventData, @@ -521,7 +521,8 @@ def __init__( self.process_runahead_limit() if self.run_mode('simulation', 'dummy'): - self.configure_sim_modes() + configure_sim_modes( + self.taskdefs.values(), self.run_mode()) self.configure_workflow_state_polling_tasks() @@ -1340,68 +1341,6 @@ def configure_workflow_state_polling_tasks(self): script = "echo " + comstr + "\n" + comstr rtc['script'] = script - def configure_sim_modes(self): - """Adjust task defs for simulation and dummy mode.""" - for tdef in self.taskdefs.values(): - # Compute simulated run time by scaling the execution limit. - rtc = tdef.rtconfig - limit = rtc['execution time limit'] - speedup = rtc['simulation']['speedup factor'] - if limit and speedup: - sleep_sec = (DurationParser().parse( - str(limit)).get_seconds() / speedup) - else: - sleep_sec = DurationParser().parse( - str(rtc['simulation']['default run length']) - ).get_seconds() - rtc['execution time limit'] = ( - sleep_sec + DurationParser().parse(str( - rtc['simulation']['time limit buffer'])).get_seconds() - ) - rtc['job']['simulated run length'] = sleep_sec - - # Generate dummy scripting. - rtc['init-script'] = "" - rtc['env-script'] = "" - rtc['pre-script'] = "" - rtc['post-script'] = "" - scr = "sleep %d" % sleep_sec - # Dummy message outputs. - for msg in rtc['outputs'].values(): - scr += "\ncylc message '%s'" % msg - if rtc['simulation']['fail try 1 only']: - arg1 = "true" - else: - arg1 = "false" - arg2 = " ".join(rtc['simulation']['fail cycle points']) - scr += "\ncylc__job__dummy_result %s %s || exit 1" % (arg1, arg2) - rtc['script'] = scr - - # Dummy mode jobs should run on platform localhost - # All Cylc 7 config items which conflict with platform are removed. - for section, keys in FORBIDDEN_WITH_PLATFORM.items(): - if section in rtc: - for key in keys: - if key in rtc[section]: - rtc[section][key] = None - - rtc['platform'] = 'localhost' - - # Disable environment, in case it depends on env-script. - rtc['environment'] = {} - - # Simulation mode tasks should fail in which cycle points? - f_pts = [] - f_pts_orig = rtc['simulation']['fail cycle points'] - if 'all' in f_pts_orig: - # None for "fail all points". - f_pts = None - else: - # (And [] for "fail no points".) - for point_str in f_pts_orig: - f_pts.append(get_point(point_str).standardise()) - rtc['simulation']['fail cycle points'] = f_pts - def get_parent_lists(self): return self.runtime['parents'] diff --git a/cylc/flow/data_messages.proto b/cylc/flow/data_messages.proto index cc56fba2e62..6068bb1c5df 100644 --- a/cylc/flow/data_messages.proto +++ b/cylc/flow/data_messages.proto @@ -105,6 +105,7 @@ message PbWorkflow { optional bool pruned = 37; optional int32 is_runahead_total = 38; optional bool states_updated = 39; + optional int32 n_edge_distance = 40; } // Selected runtime fields @@ -227,6 +228,7 @@ message PbTaskProxy { optional bool is_runahead = 26; optional bool flow_wait = 27; optional PbRuntime runtime = 28; + optional int32 graph_depth = 29; } message PbFamily { @@ -264,6 +266,7 @@ message PbFamilyProxy { optional bool is_runahead = 19; optional int32 is_runahead_total = 20; optional PbRuntime runtime = 21; + optional int32 graph_depth = 22; } message PbEdge { diff --git a/cylc/flow/data_messages_pb2.py b/cylc/flow/data_messages_pb2.py index 1bf44b36dae..82c620bcacf 100644 --- a/cylc/flow/data_messages_pb2.py +++ b/cylc/flow/data_messages_pb2.py @@ -14,7 +14,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xa2\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18 \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updated\"\xb9\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputs\"\x9d\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18 \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtimeJ\x04\x08\x1d\x10\x1e\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xe7\x07\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtime\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\x84\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtime\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xd4\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18 \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x12\x1c\n\x0fn_edge_distance\x18( \x01(\x05H\x1c\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updatedB\x12\n\x10_n_edge_distance\"\xb9\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputs\"\x9d\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18 \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtimeJ\x04\x08\x1d\x10\x1e\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\x91\x08\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x1d \x01(\x05H\x0f\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xae\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x16 \x01(\x05H\x0f\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -40,61 +40,61 @@ _globals['_PBTASKPROXYREFS']._serialized_start=349 _globals['_PBTASKPROXYREFS']._serialized_end=388 _globals['_PBWORKFLOW']._serialized_start=391 - _globals['_PBWORKFLOW']._serialized_end=1961 - _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_start=1411 - _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_end=1461 - _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_start=1463 - _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_end=1536 - _globals['_PBRUNTIME']._serialized_start=1964 - _globals['_PBRUNTIME']._serialized_end=2789 - _globals['_PBJOB']._serialized_start=2792 - _globals['_PBJOB']._serialized_end=3461 - _globals['_PBTASK']._serialized_start=3464 - _globals['_PBTASK']._serialized_end=3818 - _globals['_PBPOLLTASK']._serialized_start=3821 - _globals['_PBPOLLTASK']._serialized_end=4037 - _globals['_PBCONDITION']._serialized_start=4040 - _globals['_PBCONDITION']._serialized_end=4243 - _globals['_PBPREREQUISITE']._serialized_start=4246 - _globals['_PBPREREQUISITE']._serialized_end=4396 - _globals['_PBOUTPUT']._serialized_start=4399 - _globals['_PBOUTPUT']._serialized_end=4539 - _globals['_PBTRIGGER']._serialized_start=4542 - _globals['_PBTRIGGER']._serialized_end=4707 - _globals['_PBTASKPROXY']._serialized_start=4710 - _globals['_PBTASKPROXY']._serialized_end=5709 - _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=5335 - _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=5392 - _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=5394 - _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=5461 - _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=5463 - _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=5523 - _globals['_PBFAMILY']._serialized_start=5712 - _globals['_PBFAMILY']._serialized_end=6040 - _globals['_PBFAMILYPROXY']._serialized_start=6043 - _globals['_PBFAMILYPROXY']._serialized_end=6815 - _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_start=1411 - _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_end=1461 - _globals['_PBEDGE']._serialized_start=6818 - _globals['_PBEDGE']._serialized_end=7006 - _globals['_PBEDGES']._serialized_start=7008 - _globals['_PBEDGES']._serialized_end=7131 - _globals['_PBENTIREWORKFLOW']._serialized_start=7134 - _globals['_PBENTIREWORKFLOW']._serialized_end=7376 - _globals['_EDELTAS']._serialized_start=7379 - _globals['_EDELTAS']._serialized_end=7554 - _globals['_FDELTAS']._serialized_start=7557 - _globals['_FDELTAS']._serialized_end=7736 - _globals['_FPDELTAS']._serialized_start=7739 - _globals['_FPDELTAS']._serialized_end=7929 - _globals['_JDELTAS']._serialized_start=7932 - _globals['_JDELTAS']._serialized_end=8105 - _globals['_TDELTAS']._serialized_start=8108 - _globals['_TDELTAS']._serialized_end=8283 - _globals['_TPDELTAS']._serialized_start=8286 - _globals['_TPDELTAS']._serialized_end=8472 - _globals['_WDELTAS']._serialized_start=8475 - _globals['_WDELTAS']._serialized_end=8670 - _globals['_ALLDELTAS']._serialized_start=8673 - _globals['_ALLDELTAS']._serialized_end=8882 + _globals['_PBWORKFLOW']._serialized_end=2011 + _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_start=1441 + _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_end=1491 + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_start=1493 + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_end=1566 + _globals['_PBRUNTIME']._serialized_start=2014 + _globals['_PBRUNTIME']._serialized_end=2839 + _globals['_PBJOB']._serialized_start=2842 + _globals['_PBJOB']._serialized_end=3511 + _globals['_PBTASK']._serialized_start=3514 + _globals['_PBTASK']._serialized_end=3868 + _globals['_PBPOLLTASK']._serialized_start=3871 + _globals['_PBPOLLTASK']._serialized_end=4087 + _globals['_PBCONDITION']._serialized_start=4090 + _globals['_PBCONDITION']._serialized_end=4293 + _globals['_PBPREREQUISITE']._serialized_start=4296 + _globals['_PBPREREQUISITE']._serialized_end=4446 + _globals['_PBOUTPUT']._serialized_start=4449 + _globals['_PBOUTPUT']._serialized_end=4589 + _globals['_PBTRIGGER']._serialized_start=4592 + _globals['_PBTRIGGER']._serialized_end=4757 + _globals['_PBTASKPROXY']._serialized_start=4760 + _globals['_PBTASKPROXY']._serialized_end=5801 + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=5411 + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=5468 + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=5470 + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=5537 + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=5539 + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=5599 + _globals['_PBFAMILY']._serialized_start=5804 + _globals['_PBFAMILY']._serialized_end=6132 + _globals['_PBFAMILYPROXY']._serialized_start=6135 + _globals['_PBFAMILYPROXY']._serialized_end=6949 + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_start=1441 + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_end=1491 + _globals['_PBEDGE']._serialized_start=6952 + _globals['_PBEDGE']._serialized_end=7140 + _globals['_PBEDGES']._serialized_start=7142 + _globals['_PBEDGES']._serialized_end=7265 + _globals['_PBENTIREWORKFLOW']._serialized_start=7268 + _globals['_PBENTIREWORKFLOW']._serialized_end=7510 + _globals['_EDELTAS']._serialized_start=7513 + _globals['_EDELTAS']._serialized_end=7688 + _globals['_FDELTAS']._serialized_start=7691 + _globals['_FDELTAS']._serialized_end=7870 + _globals['_FPDELTAS']._serialized_start=7873 + _globals['_FPDELTAS']._serialized_end=8063 + _globals['_JDELTAS']._serialized_start=8066 + _globals['_JDELTAS']._serialized_end=8239 + _globals['_TDELTAS']._serialized_start=8242 + _globals['_TDELTAS']._serialized_end=8417 + _globals['_TPDELTAS']._serialized_start=8420 + _globals['_TPDELTAS']._serialized_end=8606 + _globals['_WDELTAS']._serialized_start=8609 + _globals['_WDELTAS']._serialized_end=8804 + _globals['_ALLDELTAS']._serialized_start=8807 + _globals['_ALLDELTAS']._serialized_end=9016 # @@protoc_insertion_point(module_scope) diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py index 0781020a8b5..ef77105b3a4 100644 --- a/cylc/flow/data_store_mgr.py +++ b/cylc/flow/data_store_mgr.py @@ -33,19 +33,17 @@ includes workflow, task, and family definition objects. The cycle point nodes/edges (i.e. task/family proxies) generation is triggered -individually on transition from staging to active task pool. Each active task -is generated along with any children and parents recursively out to a -specified maximum graph distance (n_edge_distance), that can be externally -altered (via API). Collectively this forms the N-Distance-Window on the -workflow graph. - -Pruning of data-store elements is done using both the collection/set of nodes -generated through the associated graph paths of the active nodes and the -tracking of the boundary nodes (n_edge_distance+1) of those active nodes. -Once active, these boundary nodes act as the prune trigger for their -original/generator node(s). Set operations are used to do a diff between the -nodes of active paths (paths whose node is in the active task pool) and the -nodes of flagged paths (whose boundary node(s) have become active). +individually on transition to active task pool. Each active task is generated +along with any children and parents via a graph walk out to a specified maximum +graph distance (n_edge_distance), that can be externally altered (via API). +Collectively this forms the N-Distance-Window on the workflow graph. + +Pruning of data-store elements is done using the collection/set of nodes +generated at the boundary of an active node's graph walk and registering active +node's parents against them. Once active, these boundary nodes act as the prune +triggers for the associated parent nodes. Set operations are used to do a diff +between the nodes of active paths (paths whose node is in the active task pool) +and the nodes of flagged paths (whose boundary node(s) have become active). Updates are created by the event/task/job managers. @@ -63,7 +61,10 @@ from time import time from typing import ( Any, + Dict, Optional, + List, + Set, TYPE_CHECKING, Tuple, Union, @@ -71,6 +72,7 @@ import zlib from cylc.flow import __version__ as CYLC_VERSION, LOG +from cylc.flow.cycling.loader import get_point from cylc.flow.data_messages_pb2 import ( # type: ignore PbEdge, PbEntireWorkflow, PbFamily, PbFamilyProxy, PbJob, PbTask, PbTaskProxy, PbWorkflow, PbRuntime, AllDeltas, EDeltas, FDeltas, @@ -481,7 +483,7 @@ def __init__(self, schd): state: deque(maxlen=LATEST_STATE_TASKS_QUEUE_SIZE) for state in TASK_STATUSES_ORDERED } - self.xtrigger_tasks = {} + self.xtrigger_tasks: Dict[str, Set[Tuple[str, str]]] = {} # Managed data types self.data = { self.workflow_id: deepcopy(DATA_TEMPLATE) @@ -504,8 +506,11 @@ def __init__(self, schd): self.all_task_pool = set() self.all_n_window_nodes = set() self.n_window_nodes = {} - self.n_window_edges = {} - self.n_window_boundary_nodes = {} + self.n_window_edges = set() + self.n_window_node_walks = {} + self.n_window_completed_walks = set() + self.n_window_depths = {} + self.update_window_depths = False self.db_load_task_proxies = {} self.family_pruned_ids = set() self.prune_trigger_nodes = {} @@ -563,6 +568,7 @@ def generate_definition_elements(self): families = self.added[FAMILIES] workflow = self.added[WORKFLOW] workflow.id = self.workflow_id + workflow.n_edge_distance = self.n_edge_distance workflow.last_updated = update_time workflow.stamp = f'{workflow.id}@{workflow.last_updated}' # Treat play/restart as hard reload of definition. @@ -708,32 +714,25 @@ def increment_graph_window( source_tokens: Tokens, point, flow_nums, - edge_distance=0, - active_id: Optional[str] = None, - descendant=False, - is_parent=False, is_manual_submit=False, itask=None ) -> None: """Generate graph window about active task proxy to n-edge-distance. - A recursive function, that creates a node then moves to children and - parents repeating this process out to one edge beyond the max window - size (in edges). Going out one edge further, we can trigger - pruning as new active tasks appear beyond this boundary. - + Fills in graph walk from existing walks if possible, otherwise expands + the graph front from whereever hasn't been walked. + Walk nodes are grouped into locations which are tag according to + parent child path, i.e. 'cpc' would be children-parents-children away + from active/start task. Which not only provide a way to cheaply rewalk, + but also the edge distance from origin. + The futherest child boundary nodes are registered as prune triggers for + the origin's parents, so when they become active the parents are + assessed for pruning eligibility. Args: source_tokens (cylc.flow.id.Tokens) point (PointBase) flow_nums (set) - edge_distance (int): - Graph distance from active/origin node. - active_id (str): - Active/origin node id. - descendant (bool): - Is the current node a direct descendent of the active/origin. - is_parent (bool) is_manual_submit (bool) itask (cylc.flow.task_proxy.TaskProxy): Active/Other task proxy, passed in with pool invocation. @@ -742,146 +741,343 @@ def increment_graph_window( None """ - is_active = not (descendant or is_parent) - # ID passed through recursion as reference to original/active node. - if active_id is None: - source_tokens = self.id_.duplicate(source_tokens) - active_id = source_tokens.id - - # flag manual triggers for pruning on deletion. - if is_manual_submit: - self.prune_trigger_nodes.setdefault(active_id, set()).add( - source_tokens.id - ) - - # Setup and check if active node is another's boundary node - # to flag its paths for pruning. - if is_active: - self.n_window_edges[active_id] = set() - self.n_window_boundary_nodes[active_id] = {} - self.n_window_nodes[active_id] = set() - if active_id in self.prune_trigger_nodes: - self.prune_flagged_nodes.update( - self.prune_trigger_nodes[active_id]) - del self.prune_trigger_nodes[active_id] - # This part is vital to constructing a set of boundary nodes - # associated with the current Active node. - if edge_distance > self.n_edge_distance: - if descendant and self.n_edge_distance > 0: - self.n_window_boundary_nodes[ - active_id - ].setdefault(edge_distance, set()).add(source_tokens.id) - return + # common refrences + active_id = source_tokens.id + all_walks = self.n_window_node_walks + taskdefs = self.schd.config.taskdefs + final_point = self.schd.config.final_point + + # walk keys/tags + # Children location tag + c_tag = 'c' + # Parents location tag + p_tag = 'p' + + # Setup walk fields: + # - locations (locs): i.e. 'cpc' children-parents-children from origin, + # with their respective node ids. + # - orphans: task no longer exists in workflow. + # - done_locs: set of locactions that have been walked over. + # - done_ids: set of node ids that have been walked (from initial + # walk filling, that may not have been the entire walk). + # If walk already completed, must have gone from non-active to active + # again.. So redo walk (as walk nodes may be pruned). + if ( + active_id not in all_walks + or active_id in self.n_window_completed_walks + ): + all_walks[active_id] = { + 'locations': {}, + 'orphans': set(), + 'done_locs': set(), + 'done_ids': set(), + 'walk_ids': {active_id}, + 'depths': { + depth: set() + for depth in range(1, self.n_edge_distance + 1) + } + } + if active_id in self.n_window_completed_walks: + self.n_window_completed_walks.remove(active_id) + active_walk = all_walks[active_id] + active_locs = active_walk['locations'] + if source_tokens['task'] not in taskdefs: + active_walk['orphans'].add(active_id) # Generate task proxy node - is_orphan, graph_children = self.generate_ghost_task( + self.n_window_nodes[active_id] = set() + + self.generate_ghost_task( source_tokens, point, flow_nums, - is_parent, + False, itask ) - self.n_window_nodes[active_id].add(source_tokens.id) - - edge_distance += 1 + # Pre-populate from previous walks + # Will check all location permutations. + # There may be short cuts for parent locs, however children will more + # likely be incomplete walks with no 'done_locs' and using parent's + # children will required sifting out cousin branches. + working_locs: List[str] = [] + if self.n_edge_distance > 1: + if c_tag in active_locs: + working_locs.extend(('cc', 'cp')) + if p_tag in active_locs: + working_locs.extend(('pp', 'pc')) + n_depth = 2 + while working_locs: + for w_loc in working_locs: + loc_done = True + # Most will be incomplete walks, however, we can check. + # i.e. parents of children may all exist. + if w_loc[:-1] in active_locs: + for loc_id in active_locs[w_loc[:-1]]: + if loc_id not in all_walks: + loc_done = False + break + else: + continue + # find child nodes of parent location, + # i.e. 'cpcc' = 'cpc' + 'c' + w_set = set().union(*( + all_walks[loc_id]['locations'][w_loc[-1]] + for loc_id in active_locs[w_loc[:-1]] + if ( + loc_id in all_walks + and w_loc[-1] in all_walks[loc_id]['locations'] + ) + )) + w_set.difference_update(active_walk['walk_ids']) + if w_set: + active_locs[w_loc] = w_set + active_walk['walk_ids'].update(w_set) + active_walk['depths'][n_depth].update(w_set) + # If child/parent nodes have been pruned we will need + # to regenerate them. + if ( + loc_done + and not w_set.difference(self.all_n_window_nodes) + ): + active_walk['done_locs'].add(w_loc[:-1]) + active_walk['done_ids'].update( + active_locs[w_loc[:-1]] + ) + working_locs = [ + new_loc + for loc in working_locs + if loc in active_locs and len(loc) < self.n_edge_distance + for new_loc in (loc + c_tag, loc + p_tag) + ] + n_depth += 1 - # Don't expand window about orphan task. + # Graph walk + node_tokens: Tokens child_tokens: Tokens parent_tokens: Tokens - if not is_orphan: - tdef = self.schd.config.taskdefs[source_tokens['task']] - # TODO: xtrigger is workflow_state edges too - # Reference set for workflow relations - final_point = self.schd.config.final_point - if descendant or is_active: - if graph_children is None: - graph_children = generate_graph_children(tdef, point) - if not any(graph_children.values()): - self.n_window_boundary_nodes[active_id].setdefault( - edge_distance - 1, - set() - ).add(source_tokens.id) - - # Children/downstream nodes - for items in graph_children.values(): - for child_name, child_point, _ in items: - if child_point > final_point: - continue - child_tokens = self.id_.duplicate( - cycle=str(child_point), - task=child_name, - ) - # We still increment the graph one further to find - # boundary nodes, but don't create elements. - if edge_distance <= self.n_edge_distance: - self.generate_edge( - source_tokens, - child_tokens, - active_id - ) - if child_tokens.id in self.n_window_nodes[active_id]: - continue - self.increment_graph_window( - child_tokens, - child_point, - flow_nums, - edge_distance, - active_id, - True, - False - ) + walk_incomplete = True + while walk_incomplete: + walk_incomplete = False + # Only walk locations not fully explored + locations = [ + loc + for loc in active_locs + if ( - # Parents/upstream nodes - if is_parent or is_active: - for items in generate_graph_parents( - tdef, - point, - self.schd.config.taskdefs - ).values(): - for parent_name, parent_point, _ in items: - if parent_point > final_point: + len(loc) < self.n_edge_distance + and loc not in active_walk['done_locs'] + ) + ] + # Origin/Active usually first or isolate nodes + if ( + not active_walk['done_ids'] + and not locations + and active_id not in active_walk['orphans'] + and self.n_edge_distance != 0 + ): + locations = [''] + # Explore/walk locations + for location in locations: + walk_incomplete = True + if not location: + loc_nodes = {active_id} + else: + loc_nodes = active_locs[location] + active_walk['done_locs'].add(location) + c_loc = location + c_tag + p_loc = location + p_tag + c_ids = set() + p_ids = set() + n_depth = len(location) + 1 + # Exclude walked nodes at this location. + # This also helps avoid walking in a circle. + for node_id in loc_nodes.difference(active_walk['done_ids']): + active_walk['done_ids'].add(node_id) + node_tokens = Tokens(node_id) + # Don't expand window about orphan task. + try: + tdef = taskdefs[node_tokens['task']] + except KeyError: + active_walk['orphans'].add(node_id) + continue + # Use existing children/parents from other walks. + # (note: nodes/edges should already be generated) + c_done = False + p_done = False + if node_id in all_walks and node_id is not active_id: + with suppress(KeyError): + # If children have been pruned, don't skip, + # re-generate them (uncommon or impossible?). + if not all_walks[node_id]['locations'][ + c_tag + ].difference(self.all_n_window_nodes): + c_ids.update( + all_walks[node_id]['locations'][c_tag] + ) + c_done = True + with suppress(KeyError): + # If parent have been pruned, don't skip, + # re-generate them (more common case). + if not all_walks[node_id]['locations'][ + p_tag + ].difference(self.all_n_window_nodes): + p_ids.update( + all_walks[node_id]['locations'][p_tag] + ) + p_done = True + if p_done and c_done: continue - parent_tokens = self.id_.duplicate( - cycle=str(parent_point), - task=parent_name, - ) - if edge_distance <= self.n_edge_distance: - # reverse for parent - self.generate_edge( - parent_tokens, - source_tokens, - active_id + + # Children/downstream nodes + # TODO: xtrigger is workflow_state edges too + # see: https://github.com/cylc/cylc-flow/issues/4582 + # Reference set for workflow relations + nc_ids = set() + if not c_done: + if itask is not None and n_depth == 1: + graph_children = itask.graph_children + else: + graph_children = generate_graph_children( + tdef, + get_point(node_tokens['cycle']) ) - if parent_tokens.id in self.n_window_nodes[active_id]: - continue - self.increment_graph_window( - parent_tokens, - parent_point, - flow_nums, - edge_distance, - active_id, - False, - True - ) + for items in graph_children.values(): + for child_name, child_point, _ in items: + if child_point > final_point: + continue + child_tokens = self.id_.duplicate( + cycle=str(child_point), + task=child_name, + ) + self.generate_ghost_task( + child_tokens, + child_point, + flow_nums, + False, + None, + n_depth + ) + self.generate_edge( + node_tokens, + child_tokens, + active_id + ) + nc_ids.add(child_tokens.id) + + # Parents/upstream nodes + np_ids = set() + if not p_done: + for items in generate_graph_parents( + tdef, + get_point(node_tokens['cycle']), + taskdefs + ).values(): + for parent_name, parent_point, _ in items: + if parent_point > final_point: + continue + parent_tokens = self.id_.duplicate( + cycle=str(parent_point), + task=parent_name, + ) + self.generate_ghost_task( + parent_tokens, + parent_point, + flow_nums, + True, + None, + n_depth + ) + # reverse for parent + self.generate_edge( + parent_tokens, + node_tokens, + active_id + ) + np_ids.add(parent_tokens.id) + + # Register new walk + if node_id not in all_walks: + all_walks[node_id] = { + 'locations': {}, + 'done_ids': set(), + 'done_locs': set(), + 'orphans': set(), + 'walk_ids': {node_id} | nc_ids | np_ids, + 'depths': { + depth: set() + for depth in range(1, self.n_edge_distance + 1) + } + } + if nc_ids: + all_walks[node_id]['locations'][c_tag] = nc_ids + all_walks[node_id]['depths'][1].update(nc_ids) + c_ids.update(nc_ids) + if np_ids: + all_walks[node_id]['locations'][p_tag] = np_ids + all_walks[node_id]['depths'][1].update(np_ids) + p_ids.update(np_ids) + + # Create location association + c_ids.difference_update(active_walk['walk_ids']) + if c_ids: + active_locs.setdefault(c_loc, set()).update(c_ids) + p_ids.difference_update(active_walk['walk_ids']) + if p_ids: + active_locs.setdefault(p_loc, set()).update(p_ids) + active_walk['walk_ids'].update(c_ids, p_ids) + active_walk['depths'][n_depth].update(c_ids, p_ids) + + self.n_window_completed_walks.add(active_id) + self.n_window_nodes[active_id].update(active_walk['walk_ids']) - # If this is the active task (edge_distance has been incremented), - # then add the most distant child as a trigger to prune it. - if is_active: - levels = self.n_window_boundary_nodes[active_id].keys() + # This part is vital to constructing a set of boundary nodes + # associated with the n=0 window of current active node. + # Only trigger pruning for furthest set of boundary nodes + boundary_nodes: Set[str] = set() + max_level: int = 0 + with suppress(ValueError): + max_level = max( + len(loc) + for loc in active_locs + if p_tag not in loc + ) + # add the most distant child as a trigger to prune it. + boundary_nodes.update(*( + active_locs[loc] + for loc in active_locs + if p_tag not in loc and len(loc) >= max_level + )) + if not boundary_nodes and not max_level: # Could be self-reference node foo:failed => foo - if not levels: - self.n_window_boundary_nodes[active_id][0] = {active_id} - levels = (0,) - # Only trigger pruning for furthest set of boundary nodes - for tp_id in self.n_window_boundary_nodes[active_id][max(levels)]: - self.prune_trigger_nodes.setdefault( - tp_id, set()).add(active_id) - del self.n_window_boundary_nodes[active_id] - if self.n_window_edges[active_id]: - getattr(self.updated[WORKFLOW], EDGES).edges.extend( - self.n_window_edges[active_id]) + boundary_nodes = {active_id} + # associate + for tp_id in boundary_nodes: + try: + self.prune_trigger_nodes.setdefault(tp_id, set()).update( + active_walk['walk_ids'] + ) + self.prune_trigger_nodes[tp_id].discard(tp_id) + except KeyError: + self.prune_trigger_nodes.setdefault(tp_id, set()).add( + active_id + ) + # flag manual triggers for pruning on deletion. + if is_manual_submit: + self.prune_trigger_nodes.setdefault(active_id, set()).add( + active_id + ) + if active_walk['orphans']: + self.prune_trigger_nodes.setdefault(active_id, set()).union( + active_walk['orphans'] + ) + # Check if active node is another's boundary node + # to flag its paths for pruning. + if active_id in self.prune_trigger_nodes: + self.prune_flagged_nodes.update( + self.prune_trigger_nodes[active_id]) + del self.prune_trigger_nodes[active_id] def generate_edge( self, @@ -892,7 +1088,7 @@ def generate_edge( """Construct edge of child and parent task proxy node.""" # Initiate edge element. e_id = self.edge_id(parent_tokens, child_tokens) - if e_id in self.n_window_edges[active_id]: + if e_id in self.n_window_edges: return if ( e_id not in self.data[self.workflow_id][EDGES] @@ -910,7 +1106,8 @@ def generate_edge( self.updated[TASK_PROXIES].setdefault( parent_tokens.id, PbTaskProxy(id=parent_tokens.id)).edges.append(e_id) - self.n_window_edges[active_id].add(e_id) + getattr(self.updated[WORKFLOW], EDGES).edges.append(e_id) + self.n_window_edges.add(e_id) def remove_pool_node(self, name, point): """Remove ID reference and flag isolate node/branch for pruning.""" @@ -928,13 +1125,16 @@ def remove_pool_node(self, name, point): ): self.prune_flagged_nodes.update(self.prune_trigger_nodes[tp_id]) del self.prune_trigger_nodes[tp_id] - self.updates_pending = True elif ( tp_id in self.n_window_nodes and self.n_window_nodes[tp_id].isdisjoint(self.all_task_pool) ): self.prune_flagged_nodes.add(tp_id) - self.updates_pending = True + elif tp_id in self.n_window_node_walks: + self.prune_flagged_nodes.update( + self.n_window_node_walks[tp_id]['walk_ids'] + ) + self.updates_pending = True def add_pool_node(self, name, point): """Add external ID reference for internal task pool node.""" @@ -943,6 +1143,7 @@ def add_pool_node(self, name, point): task=name, ).id self.all_task_pool.add(tp_id) + self.update_window_depths = True def generate_ghost_task( self, @@ -950,8 +1151,9 @@ def generate_ghost_task( point, flow_nums, is_parent=False, - itask=None - ) -> Tuple[bool, Optional[dict]]: + itask=None, + n_depth=0, + ): """Create task-point element populated with static data. Args: @@ -962,29 +1164,26 @@ def generate_ghost_task( Used to determine whether to load DB state. itask (cylc.flow.task_proxy.TaskProxy): Update task-node from corresponding task proxy object. + n_depth (int): n-window graph edge distance. Returns: - (is_orphan, graph_children) - Orphan tasks with no children return (True, None) respectively. + None """ + tp_id = tokens.id + if ( + tp_id in self.data[self.workflow_id][TASK_PROXIES] + or tp_id in self.added[TASK_PROXIES] + ): + return + name = tokens['task'] point_string = tokens['cycle'] t_id = self.definition_id(name) - tp_id = tokens.id - task_proxies = self.data[self.workflow_id][TASK_PROXIES] - - is_orphan = False - if name not in self.schd.config.taskdefs: - is_orphan = True if itask is None: itask = self.schd.pool.get_task(point_string, name) - if tp_id in task_proxies or tp_id in self.added[TASK_PROXIES]: - if itask is None: - return is_orphan, None - return is_orphan, itask.graph_children if itask is None: itask = TaskProxy( @@ -996,7 +1195,9 @@ def generate_ghost_task( data_mode=True ) - if is_orphan: + is_orphan = False + if name not in self.schd.config.taskdefs: + is_orphan = True self.generate_orphan_task(itask) # Most of the time the definition node will be in the store. @@ -1007,7 +1208,7 @@ def generate_ghost_task( task_def = self.added[TASKS][t_id] except KeyError: # Task removed from workflow definition. - return False, itask.graph_children + return update_time = time() tp_stamp = f'{tp_id}@{update_time}' @@ -1021,8 +1222,11 @@ def generate_ghost_task( in self.schd.pool.tasks_to_hold ), depth=task_def.depth, + graph_depth=n_depth, name=name, ) + self.all_n_window_nodes.add(tp_id) + self.n_window_depths.setdefault(n_depth, set()).add(tp_id) tproxy.namespace[:] = task_def.namespace if is_orphan: @@ -1075,7 +1279,7 @@ def generate_ghost_task( self.updates_pending = True - return is_orphan, itask.graph_children + return def generate_orphan_task(self, itask): """Generate orphan task definition.""" @@ -1194,7 +1398,6 @@ def generate_ghost_family(self, fp_id, child_fam=None, child_task=None): def apply_task_proxy_db_history(self): """Extract and apply DB history on given task proxies.""" - if not self.db_load_task_proxies: return @@ -1315,7 +1518,7 @@ def _process_internal_task_proxy(self, itask, tproxy): xtrig.id = sig xtrig.label = label xtrig.satisfied = satisfied - self.xtrigger_tasks.setdefault(sig, set()).add(tproxy.id) + self.xtrigger_tasks.setdefault(sig, set()).add((tproxy.id, label)) if tproxy.state in self.latest_state_tasks: tp_ref = itask.identity @@ -1384,7 +1587,7 @@ def insert_job(self, name, cycle_point, status, job_conf): name=tproxy.name, cycle_point=tproxy.cycle_point, execution_time_limit=job_conf.get('execution_time_limit'), - platform=job_conf.get('platform')['name'], + platform=job_conf['platform']['name'], job_runner_name=job_conf.get('job_runner_name'), ) # Not all fields are populated with some submit-failures, @@ -1499,16 +1702,23 @@ def insert_db_job(self, row_idx, row): def update_data_structure(self): """Workflow batch updates in the data structure.""" - # load database history for flagged nodes - self.apply_task_proxy_db_history() # Avoids changing window edge distance during edge/node creation if self.next_n_edge_distance is not None: self.n_edge_distance = self.next_n_edge_distance + self.window_resize_rewalk() self.next_n_edge_distance = None + # load database history for flagged nodes + self.apply_task_proxy_db_history() + self.updates_pending_follow_on = False self.prune_data_store() + + # Find depth changes and create deltas + if self.update_window_depths: + self.window_depth_finder() + if self.updates_pending: # update self.update_family_proxies() @@ -1547,6 +1757,83 @@ def update_workflow_states(self): self.apply_delta_checksum() self.publish_deltas = self.get_publish_deltas() + def window_resize_rewalk(self): + """Re-create data-store n-window on resize.""" + tokens: Tokens + # Gather pre-resize window nodes + if not self.all_n_window_nodes: + self.all_n_window_nodes = set().union(*( + v + for k, v in self.n_window_nodes.items() + if k in self.all_task_pool + )) + + # Clear window walks, and walk from scratch. + self.prune_flagged_nodes.clear() + self.n_window_node_walks.clear() + for tp_id in self.all_task_pool: + tokens = Tokens(tp_id) + tp_id, tproxy = self.store_node_fetcher(tokens) + self.increment_graph_window( + tokens, + get_point(tokens['cycle']), + tproxy.flow_nums + ) + # Flag difference between old and new window for pruning. + self.prune_flagged_nodes.update( + self.all_n_window_nodes.difference(*( + v + for k, v in self.n_window_nodes.items() + if k in self.all_task_pool + )) + ) + self.update_window_depths = True + + def window_depth_finder(self): + """Recalculate window depths, creating depth deltas.""" + # Setup new window depths + n_window_depths: Dict[int, Set[str]] = { + 0: self.all_task_pool.copy() + } + + depth = 1 + # Since starting from smaller depth, exclude those whose depth has + # already been found. + depth_found_tasks: Set[str] = self.all_task_pool.copy() + while depth <= self.n_edge_distance: + n_window_depths[depth] = set().union(*( + self.n_window_node_walks[n_id]['depths'][depth] + for n_id in self.all_task_pool + if ( + n_id in self.n_window_node_walks + and depth in self.n_window_node_walks[n_id]['depths'] + ) + )).difference(depth_found_tasks) + depth_found_tasks.update(n_window_depths[depth]) + # Calculate next depth parameters. + depth += 1 + + # Create deltas of those whose depth has changed, a node should only + # appear once across all depths. + # So the diff will only contain it at a single depth and if it didn't + # appear at the same depth previously. + update_time = time() + for depth, node_set in n_window_depths.items(): + node_set_diff = node_set.difference( + self.n_window_depths.setdefault(depth, set()) + ) + if not self.updates_pending and node_set_diff: + self.updates_pending = True + for tp_id in node_set_diff: + tp_delta = self.updated[TASK_PROXIES].setdefault( + tp_id, PbTaskProxy(id=tp_id) + ) + tp_delta.stamp = f'{tp_id}@{update_time}' + tp_delta.graph_depth = depth + # Set old to new. + self.n_window_depths = n_window_depths + self.update_window_depths = False + def prune_data_store(self): """Remove flagged nodes and edges not in the set of active paths.""" @@ -1581,8 +1868,6 @@ def prune_data_store(self): for tp_id in list(node_ids): if tp_id in self.n_window_nodes: del self.n_window_nodes[tp_id] - if tp_id in self.n_window_edges: - del self.n_window_edges[tp_id] if tp_id in tp_data: node = tp_data[tp_id] elif tp_id in tp_added: @@ -1590,8 +1875,15 @@ def prune_data_store(self): else: node_ids.remove(tp_id) continue + self.n_window_edges.difference_update(node.edges) + if tp_id in self.n_window_node_walks: + del self.n_window_node_walks[tp_id] + if tp_id in self.n_window_completed_walks: + self.n_window_completed_walks.remove(tp_id) for sig in node.xtriggers: - self.xtrigger_tasks[sig].remove(tp_id) + self.xtrigger_tasks[sig].remove( + (tp_id, node.xtriggers[sig].label) + ) if not self.xtrigger_tasks[sig]: del self.xtrigger_tasks[sig] @@ -1742,6 +2034,7 @@ def _family_ascent_point_update(self, fp_id): is_held_total = 0 is_queued_total = 0 is_runahead_total = 0 + graph_depth = self.n_edge_distance for child_id in fam_node.child_families: child_node = fp_updated.get(child_id, fp_data.get(child_id)) if child_node is not None: @@ -1749,6 +2042,8 @@ def _family_ascent_point_update(self, fp_id): is_queued_total += child_node.is_queued_total is_runahead_total += child_node.is_runahead_total state_counter += Counter(dict(child_node.state_totals)) + if child_node.graph_depth < graph_depth: + graph_depth = child_node.graph_depth # Gather all child task states task_states = [] for tp_id in fam_node.child_tasks: @@ -1783,6 +2078,12 @@ def _family_ascent_point_update(self, fp_id): if tp_runahead.is_runahead: is_runahead_total += 1 + tp_depth = tp_delta + if tp_depth is None or not tp_depth.HasField('graph_depth'): + tp_depth = tp_node + if tp_depth.graph_depth < graph_depth: + graph_depth = tp_depth.graph_depth + state_counter += Counter(task_states) # created delta data element fp_delta = PbFamilyProxy( @@ -1794,7 +2095,8 @@ def _family_ascent_point_update(self, fp_id): is_queued=(is_queued_total > 0), is_queued_total=is_queued_total, is_runahead=(is_runahead_total > 0), - is_runahead_total=is_runahead_total + is_runahead_total=is_runahead_total, + graph_depth=graph_depth ) fp_delta.states[:] = state_counter.keys() # Use all states to clean up pruned counts @@ -1816,8 +2118,9 @@ def set_graph_window_extent(self, n_edge_distance): Maximum edge distance from active node. """ - self.next_n_edge_distance = n_edge_distance - self.updates_pending = True + if n_edge_distance != self.n_edge_distance: + self.next_n_edge_distance = n_edge_distance + self.updates_pending = True def update_workflow(self, reloaded=False): """Update workflow element status and state totals.""" @@ -1878,6 +2181,10 @@ def update_workflow(self, reloaded=False): if reloaded is not w_data.reloaded: w_delta.reloaded = reloaded + if w_data.n_edge_distance != self.n_edge_distance: + w_delta.n_edge_distance = self.n_edge_distance + delta_set = True + if self.schd.pool.main_pool: pool_points = set(self.schd.pool.main_pool) oldest_point = str(min(pool_points)) @@ -2175,6 +2482,7 @@ def delta_task_ext_trigger( tp_id, PbTaskProxy(id=tp_id)) tp_delta.stamp = f'{tp_id}@{update_time}' ext_trigger = tp_delta.external_triggers[trig] + ext_trigger.id = tproxy.external_triggers[trig].id ext_trigger.message = message ext_trigger.satisfied = satisfied ext_trigger.time = update_time @@ -2192,12 +2500,14 @@ def delta_task_xtrigger(self, sig, satisfied): """ update_time = time() - for tp_id in self.xtrigger_tasks.get(sig, set()): + for tp_id, label in self.xtrigger_tasks.get(sig, set()): # update task instance tp_delta = self.updated[TASK_PROXIES].setdefault( tp_id, PbTaskProxy(id=tp_id)) tp_delta.stamp = f'{tp_id}@{update_time}' xtrigger = tp_delta.xtriggers[sig] + xtrigger.id = sig + xtrigger.label = label xtrigger.satisfied = satisfied xtrigger.time = update_time self.updates_pending = True diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings index 15de1f8ea13..53b4aa2c17a 100644 --- a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings +++ b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings @@ -7,6 +7,5 @@ [[[environment]]] # These environment variables ensure that tasks can # run in the same environment as the workflow: - {% from "sys" import path, executable %} - PYTHONPATH = {{':'.join(path)}} + {% from "sys" import executable %} PATH = $(dirname {{executable}}):$PATH diff --git a/cylc/flow/etc/tutorial/runtime-tutorial/flow.cylc b/cylc/flow/etc/tutorial/runtime-tutorial/flow.cylc index 4593c75a68e..5356e113973 100644 --- a/cylc/flow/etc/tutorial/runtime-tutorial/flow.cylc +++ b/cylc/flow/etc/tutorial/runtime-tutorial/flow.cylc @@ -3,12 +3,8 @@ UTC mode = True [scheduling] - # Start the workflow 7 hours before now ignoring minutes and seconds - # * previous(T-00) takes the current time ignoring minutes and seconds. - # * - PT7H subtracts 7 hours from the time. - initial cycle point = previous(T-00) - PT7H - # Stop the workflow 6 hours after the initial cycle point. - final cycle point = +PT6H + # TODO: Set initial cycle point + # TODO: Set final cycle point [[graph]] # Repeat every three hours starting at the initial cycle point. PT3H = """ @@ -34,13 +30,5 @@ """ [runtime] - [[get_observations_camborne]] - [[get_observations_heathrow]] - [[get_observations_aldergrove]] - [[get_observations_shetland]] - [[consolidate_observations]] - [[forecast]] - [[get_rainfall]] - [[post_process_exeter]] -{% include 'etc/python-job.settings' %} +%include 'etc/python-job.settings' diff --git a/cylc/flow/graph_parser.py b/cylc/flow/graph_parser.py index f37954712c0..ad0ec280a3d 100644 --- a/cylc/flow/graph_parser.py +++ b/cylc/flow/graph_parser.py @@ -23,7 +23,8 @@ Dict, List, Tuple, - Optional + Optional, + Union ) import cylc.flow.flags @@ -85,10 +86,10 @@ class GraphParser: store dependencies for the whole workflow (call parse_graph multiple times and key results by graph section). - The general form of a dependency is "EXPRESSION => NODE", where: - * On the right, NODE is a task or family name + The general form of a dependency is "LHS => RHS", where: * On the left, an EXPRESSION of nodes involving parentheses, and logical operators '&' (AND), and '|' (OR). + * On the right, an EXPRESSION of nodes NOT involving '|' * Node names may be parameterized (any number of parameters): NODE NODE # specific parameter value @@ -517,32 +518,33 @@ def _proc_dep_pair( "Suicide markers must be" f" on the right of a trigger: {left}") + # Check that parentheses match. + mismatch_msg = 'Mismatched parentheses in: "{}"' + if left and left.count("(") != left.count(")"): + raise GraphParseError(mismatch_msg.format(left)) + if right.count("(") != right.count(")"): + raise GraphParseError(mismatch_msg.format(right)) + # Ignore cycle point offsets on the right side. # (Note we can't ban this; all nodes get process as left and right.) if '[' in right: return - # Check that parentheses match. - if left and left.count("(") != left.count(")"): - raise GraphParseError( - "Mismatched parentheses in: \"" + left + "\"") - # Split right side on AND. rights = right.split(self.__class__.OP_AND) if '' in rights or right and not all(rights): raise GraphParseError( f"Null task name in graph: {left} => {right}") + lefts: Union[List[str], List[Optional[str]]] if not left or (self.__class__.OP_OR in left or '(' in left): - # Treat conditional or bracketed expressions as a single entity. + # Treat conditional or parenthesised expressions as a single entity # Can get [None] or [""] here - lefts: List[Optional[str]] = [left] + lefts = [left] else: # Split non-conditional left-side expressions on AND. # Can get [""] here too - # TODO figure out how to handle this wih mypy: - # assign List[str] to List[Optional[str]] - lefts = left.split(self.__class__.OP_AND) # type: ignore + lefts = left.split(self.__class__.OP_AND) if '' in lefts or left and not all(lefts): raise GraphParseError( f"Null task name in graph: {left} => {right}") @@ -847,9 +849,14 @@ def _compute_triggers( trigs += [f"{name}{offset}:{trigger}"] for right in rights: + right = right.strip('()') # parentheses don't matter m = self.__class__.REC_RHS_NODE.match(right) - # This will match, bad nodes are detected earlier (type ignore): - suicide_char, name, output, opt_char = m.groups() # type: ignore + if not m: + # Bad nodes should have been detected earlier; fail loudly + raise ValueError( # pragma: no cover + f"Unexpected graph expression: '{right}'" + ) + suicide_char, name, output, opt_char = m.groups() suicide = (suicide_char == self.__class__.SUICIDE) optional = (opt_char == self.__class__.OPTIONAL) if output: diff --git a/cylc/flow/id.py b/cylc/flow/id.py index b8f34fb217f..222d16f82b6 100644 --- a/cylc/flow/id.py +++ b/cylc/flow/id.py @@ -497,7 +497,7 @@ def duplicate( )? (?: # cycle/task/job - { RELATIVE_PATTERN } + {RELATIVE_PATTERN} )? )? )? diff --git a/cylc/flow/install_plugins/log_vc_info.py b/cylc/flow/install_plugins/log_vc_info.py index d2379c5cc0d..29d861f7654 100644 --- a/cylc/flow/install_plugins/log_vc_info.py +++ b/cylc/flow/install_plugins/log_vc_info.py @@ -63,12 +63,22 @@ from pathlib import Path from subprocess import Popen, DEVNULL, PIPE from typing import ( - Any, Dict, Iterable, List, Optional, TYPE_CHECKING, TextIO, Union, overload + Any, + Dict, + Iterable, + List, + Optional, + TYPE_CHECKING, + TextIO, + Union, + overload, ) from cylc.flow import LOG as _LOG, LoggerAdaptor from cylc.flow.exceptions import CylcError import cylc.flow.flags +from cylc.flow.pipe_poller import pipe_poller +from cylc.flow.util import format_cmd from cylc.flow.workflow_files import WorkflowFiles if TYPE_CHECKING: @@ -171,7 +181,7 @@ def get_vc_info(path: Union[Path, str]) -> Optional[Dict[str, Any]]: ): LOG.debug(f"Source dir {path} is not a {vcs} repository") elif cylc.flow.flags.verbosity > -1: - LOG.warning(f"$ {vcs} {' '.join(args)}\n{exc}") + LOG.warning(f"$ {vcs} {format_cmd(args)}\n{exc}") continue info['version control system'] = vcs @@ -217,9 +227,7 @@ def _run_cmd( args: The args to pass to the version control command. cwd: Directory to run the command in. stdout: Where to redirect output (either PIPE or a - text stream/file object). Note: only use PIPE for - commands that will not generate a large output, otherwise - the pipe might get blocked. + text stream/file object). Returns: Stdout output if stdout=PIPE, else None as the output has been @@ -231,6 +239,7 @@ def _run_cmd( OSError: Non-zero return code for VCS command. """ cmd = [vcs, *args] + LOG.debug(f'$ {format_cmd(cmd)}') try: proc = Popen( # nosec cmd, @@ -245,13 +254,15 @@ def _run_cmd( # This will only be raised if the VCS command is not installed, # otherwise Popen() will succeed with a non-zero return code raise VCSNotInstalledError(vcs, exc) - ret_code = proc.wait() - out, err = proc.communicate() - if ret_code: + if stdout == PIPE: + out, err = pipe_poller(proc, proc.stdout, proc.stderr) + else: + out, err = proc.communicate() + if proc.returncode: if any(err.lower().startswith(msg) for msg in NO_BASE_ERRS[vcs]): # No base commit in repo raise VCSMissingBaseError(vcs, cwd) - raise OSError(ret_code, err) + raise OSError(proc.returncode, err) return out diff --git a/cylc/flow/job_file.py b/cylc/flow/job_file.py index 86e65e15a8e..930331dc5a4 100644 --- a/cylc/flow/job_file.py +++ b/cylc/flow/job_file.py @@ -28,13 +28,6 @@ from cylc.flow.log_level import verbosity_to_env from cylc.flow.config import interpolate_template, ParamExpandError -# the maximum number of task dependencies which Cylc will list before -# omitting the CYLC_TASK_DEPENDENCIES environment variable -# see: https://github.com/cylc/cylc-flow/issues/5551 -# NOTE: please update `src/reference/job-script-vars/var-list.txt` -# in cylc-doc if changing this value -MAX_CYLC_TASK_DEPENDENCIES_LEN = 50 - class JobFileWriter: @@ -227,18 +220,6 @@ def _write_task_environment(self, handle, job_conf): handle.write( '\n export CYLC_TASK_NAMESPACE_HIERARCHY="%s"' % ' '.join(job_conf['namespace_hierarchy'])) - if len(job_conf['dependencies']) <= MAX_CYLC_TASK_DEPENDENCIES_LEN: - handle.write( - '\n export CYLC_TASK_DEPENDENCIES="%s"' % - ' '.join(job_conf['dependencies'])) - else: - # redact the CYLC_TASK_DEPENDENCIES variable but leave a note - # explaining why - # see: https://github.com/cylc/cylc-flow/issues/5551 - handle.write( - '\n # CYLC_TASK_DEPENDENCIES=disabled' - f' (more than {MAX_CYLC_TASK_DEPENDENCIES_LEN} dependencies)' - ) handle.write( '\n export CYLC_TASK_TRY_NUMBER=%s' % job_conf['try_num']) handle.write( diff --git a/cylc/flow/network/resolvers.py b/cylc/flow/network/resolvers.py index 55016776bfb..c51946ff188 100644 --- a/cylc/flow/network/resolvers.py +++ b/cylc/flow/network/resolvers.py @@ -267,6 +267,10 @@ def node_filter(node, node_type, args, state): args.get('maxdepth', -1) < 0 or node.depth <= args['maxdepth'] ) + and ( + args.get('graph_depth', -1) < 0 + or node.graph_depth <= args['graph_depth'] + ) # Now filter node against id arg lists and ( not args.get('ids') diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py index 54222a510c7..c2202f3f31e 100644 --- a/cylc/flow/network/scan.py +++ b/cylc/flow/network/scan.py @@ -50,10 +50,8 @@ import re from typing import AsyncGenerator, Dict, Iterable, List, Optional, Tuple, Union -from pkg_resources import ( - parse_requirements, - parse_version -) +from packaging.version import parse as parse_version +from packaging.specifiers import SpecifierSet from cylc.flow import LOG from cylc.flow.async_util import ( @@ -354,11 +352,7 @@ async def validate_contact_info(flow): def parse_requirement(requirement_string): """Parse a requirement from a requirement string.""" - # we have to give the requirement a name but what we call it doesn't - # actually matter - for req in parse_requirements(f'x {requirement_string}'): - # there should only be one requirement - return (req,), {} + return (SpecifierSet(requirement_string),), {} @pipe(preproc=parse_requirement) @@ -373,7 +367,7 @@ async def cylc_version(flow, requirement): flow (dict): Flow information dictionary, provided by scan through the pipe. requirement (str): - Requirement specifier in pkg_resources format e.g. ``> 8, < 9`` + Requirement specifier in PEP 440 format e.g. ``> 8, < 9`` """ return parse_version(flow[ContactFileFields.VERSION]) in requirement @@ -391,7 +385,7 @@ async def api_version(flow, requirement): flow (dict): Flow information dictionary, provided by scan through the pipe. requirement (str): - Requirement specifier in pkg_resources format e.g. ``> 8, < 9`` + Requirement specifier in PEP 440 format e.g. ``> 8, < 9`` """ return parse_version(flow[ContactFileFields.API]) in requirement diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py index 63d9f236d7a..f9ed95c7158 100644 --- a/cylc/flow/network/schema.py +++ b/cylc/flow/network/schema.py @@ -203,6 +203,7 @@ class SortArgs(InputObjectType): 'is_runahead': Boolean(), 'mindepth': Int(default_value=-1), 'maxdepth': Int(default_value=-1), + 'graph_depth': Int(default_value=-1), 'sort': SortArgs(default_value=None), } @@ -218,6 +219,7 @@ class SortArgs(InputObjectType): 'is_runahead': Boolean(), 'mindepth': Int(default_value=-1), 'maxdepth': Int(default_value=-1), + 'graph_depth': Int(default_value=-1), 'sort': SortArgs(default_value=None), } @@ -226,8 +228,6 @@ class SortArgs(InputObjectType): 'exids': graphene.List(ID, default_value=[]), 'states': graphene.List(String, default_value=[]), 'exstates': graphene.List(String, default_value=[]), - 'mindepth': Int(default_value=-1), - 'maxdepth': Int(default_value=-1), 'sort': SortArgs(default_value=None), } @@ -785,6 +785,12 @@ class Meta: description='Any active workflow broadcasts.' ) pruned = Boolean() # TODO: what is this? write description + n_edge_distance = Int( + description=sstrip(''' + The maximum graph distance (n) from an active node + of the data-store graph window. + '''), + ) class RuntimeSetting(ObjectType): @@ -1067,6 +1073,11 @@ class Meta: depth = Int( description='The family inheritance depth', ) + graph_depth = Int( + description=sstrip(''' + The n-window graph edge depth from closet active task(s). + '''), + ) job_submits = Int( description='The number of job submissions for this task instance.', ) @@ -1217,6 +1228,12 @@ class Meta: is_runahead = Boolean() is_runahead_total = Int() depth = Int() + graph_depth = Int( + description=sstrip(''' + The n-window graph edge smallest child task/family depth + from closet active task(s). + '''), + ) child_tasks = graphene.List( TaskProxy, description="""Descendant task proxies.""", diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py index ee84a477da7..b83ff45aab9 100644 --- a/cylc/flow/option_parsers.py +++ b/cylc/flow/option_parsers.py @@ -48,6 +48,7 @@ ) WORKFLOW_ID_ARG_DOC = ('WORKFLOW', 'Workflow ID') +OPT_WORKFLOW_ID_ARG_DOC = ('[WORKFLOW]', 'Workflow ID') WORKFLOW_ID_MULTI_ARG_DOC = ('WORKFLOW ...', 'Workflow ID(s)') WORKFLOW_ID_OR_PATH_ARG_DOC = ('WORKFLOW | PATH', 'Workflow ID or path') ID_MULTI_ARG_DOC = ('ID ...', 'Workflow/Cycle/Family/Task ID(s)') diff --git a/cylc/flow/parsec/fileparse.py b/cylc/flow/parsec/fileparse.py index f4f8d723f0b..42e8a4aa40b 100644 --- a/cylc/flow/parsec/fileparse.py +++ b/cylc/flow/parsec/fileparse.py @@ -277,7 +277,7 @@ def process_plugins(fpath, opts): try: # If you want it to work on sourcedirs you need to get the options # to here. - plugin_result = entry_point.resolve()( + plugin_result = entry_point.load()( srcdir=fpath, opts=opts ) except Exception as exc: diff --git a/cylc/flow/pipe_poller.py b/cylc/flow/pipe_poller.py new file mode 100644 index 00000000000..d769f77c469 --- /dev/null +++ b/cylc/flow/pipe_poller.py @@ -0,0 +1,73 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Utility for preventing pipes from getting clogged up. + +If you're reading files from Popen (i.e. to extract command output) where the +command output has the potential to be long-ish, then you should use this +function to protect against the buffer filling up. + +Note, there is a more advanced version of this baked into the subprocpool. +""" + +from select import select + + +def pipe_poller(proc, *files, chunk_size=4096): + """Read from a process without hitting buffer issues. + + Standin for subprocess.Popen.communicate. + + When PIPE'ing from subprocesses, the output goes into a buffer. If the + buffer gets full, the subprocess will hang trying to write to it. + + This function polls the process, reading output from the buffers into + memory to prevent them from filling up. + + Args: + proc: + The process to poll. + files: + The files you want to read from, likely anything you've directed to + PIPE. + chunk_size: + The amount of text to read from the buffer on each pass. + + Returns: + tuple - The text read from each of the files in the order they were + specified. + + """ + _files = { + file: b'' if 'b' in getattr(file, 'mode', 'r') else '' + for file in files + } + + def _read(timeout=1.0): + # read any data from files + nonlocal chunk_size, files + for file in select(list(files), [], [], timeout)[0]: + buffer = file.read(chunk_size) + if len(buffer) > 0: + _files[file] += buffer + + while proc.poll() is None: + # read from the buffers + _read() + # double check the buffers now that the process has finished + _read(timeout=0.01) + + return tuple(_files.values()) diff --git a/cylc/flow/remote.py b/cylc/flow/remote.py index fd51934af11..e0ab7544a39 100644 --- a/cylc/flow/remote.py +++ b/cylc/flow/remote.py @@ -298,7 +298,8 @@ def construct_ssh_cmd( 'CYLC_CONF_PATH', 'CYLC_COVERAGE', 'CLIENT_COMMS_METH', - 'CYLC_ENV_NAME' + 'CYLC_ENV_NAME', + *platform['ssh forward environment variables'], ]: if envvar in os.environ: command.append( diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py index 1d9fc5c21dd..effe02e910d 100644 --- a/cylc/flow/scheduler.py +++ b/cylc/flow/scheduler.py @@ -113,6 +113,7 @@ ) from cylc.flow.profiler import Profiler from cylc.flow.resources import get_resources +from cylc.flow.simulation import sim_time_check from cylc.flow.subprocpool import SubProcPool from cylc.flow.templatevars import eval_var from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager @@ -1740,7 +1741,11 @@ async def main_loop(self) -> None: self.pool.set_expired_tasks() self.release_queued_tasks() - if self.pool.sim_time_check(self.message_queue): + if ( + self.pool.config.run_mode('simulation') + and sim_time_check( + self.message_queue, self.pool.get_tasks()) + ): # A simulated task state change occurred. self.reset_inactivity_timer() diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py index 5c58b1eddba..61fd408089b 100644 --- a/cylc/flow/scheduler_cli.py +++ b/cylc/flow/scheduler_cli.py @@ -25,7 +25,7 @@ import sys from typing import TYPE_CHECKING -from pkg_resources import parse_version +from packaging.version import Version from cylc.flow import LOG, __version__ from cylc.flow.exceptions import ( @@ -37,7 +37,7 @@ from cylc.flow.id import upgrade_legacy_ids from cylc.flow.host_select import select_workflow_host from cylc.flow.hostuserutil import is_remote_host -from cylc.flow.id_cli import parse_ids +from cylc.flow.id_cli import parse_ids_async from cylc.flow.loggingutil import ( close_log, RotatingLogFileHandler, @@ -354,7 +354,11 @@ def _open_logs(id_: str, no_detach: bool, restart_num: int) -> None: ) -def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None: +async def scheduler_cli( + options: 'Values', + workflow_id_raw: str, + parse_workflow_id: bool = True +) -> None: """Run the workflow. This function should contain all of the command line facing @@ -368,15 +372,18 @@ def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None: # Parse workflow name but delay Cylc 7 suite.rc deprecation warning # until after the start-up splash is printed. # TODO: singleton - (workflow_id,), _ = parse_ids( - workflow_id_raw, - constraint='workflows', - max_workflows=1, - # warn_depr=False, # TODO - ) + if parse_workflow_id: + (workflow_id,), _ = await parse_ids_async( + workflow_id_raw, + constraint='workflows', + max_workflows=1, + # warn_depr=False, # TODO + ) + else: + workflow_id = workflow_id_raw # resume the workflow if it is already running - _resume(workflow_id, options) + await _resume(workflow_id, options) # check the workflow can be safely restarted with this version of Cylc db_file = Path(get_workflow_srv_dir(workflow_id), 'db') @@ -400,9 +407,7 @@ def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None: # NOTE: asyncio.run opens an event loop, runs your coro, # then shutdown async generators and closes the event loop scheduler = Scheduler(workflow_id, options) - asyncio.run( - _setup(scheduler) - ) + await _setup(scheduler) # daemonize if requested # NOTE: asyncio event loops cannot persist across daemonization @@ -419,9 +424,14 @@ def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None: ) # run the workflow - ret = asyncio.run( - _run(scheduler) - ) + if options.no_detach: + ret = await _run(scheduler) + else: + # Note: The daemonization messes with asyncio so we have to start a + # new event loop if detaching + ret = asyncio.run( + _run(scheduler) + ) # exit # NOTE: we must clean up all asyncio / threading stuff before exiting @@ -432,7 +442,7 @@ def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None: sys.exit(ret) -def _resume(workflow_id, options): +async def _resume(workflow_id, options): """Resume the workflow if it is already running.""" try: detect_old_contact_file(workflow_id) @@ -448,7 +458,7 @@ def _resume(workflow_id, options): 'wFlows': [workflow_id] } } - pclient('graphql', mutation_kwargs) + await pclient.async_request('graphql', mutation_kwargs) sys.exit(0) except CylcError as exc: LOG.error(exc) @@ -468,7 +478,7 @@ def _version_check( if not db_file.is_file(): # not a restart return True - this_version = parse_version(__version__) + this_version = Version(__version__) last_run_version = WorkflowDatabaseManager.check_db_compatibility(db_file) for itt, (this, that) in enumerate(zip_longest( @@ -651,4 +661,4 @@ def _play(parser: COP, options: 'Values', id_: str): *options.starttask, relative=True, ) - return scheduler_cli(options, id_) + return asyncio.run(scheduler_cli(options, id_)) diff --git a/cylc/flow/scripts/completion_server.py b/cylc/flow/scripts/completion_server.py index 2ac87f87b1b..ead311955da 100644 --- a/cylc/flow/scripts/completion_server.py +++ b/cylc/flow/scripts/completion_server.py @@ -46,10 +46,8 @@ import sys import typing as t -from pkg_resources import ( - parse_requirements, - parse_version -) +from packaging.version import parse as parse_version +from packaging.specifiers import SpecifierSet from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.id import tokenise, IDTokens, Tokens @@ -74,7 +72,7 @@ # set the compatibility range for completion scripts with this server # I.E. if we change the server interface, change this compatibility range. # User's will be presented with an upgrade notice if this happens. -REQUIRED_SCRIPT_VERSION = 'completion-script >=1.0.0, <2.0.0' +REQUIRED_SCRIPT_VERSION = '>=1.0.0, <2.0.0' # register the psudo "help" and "version" commands COMMAND_LIST = list(COMMANDS) + ['help', 'version'] @@ -317,7 +315,7 @@ def list_options(command: str) -> t.List[str]: if command in COMMAND_OPTION_MAP: return COMMAND_OPTION_MAP[command] try: - entry_point = COMMANDS[command].resolve() + entry_point = COMMANDS[command].load() except KeyError: return [] parser = entry_point.parser_function() @@ -637,15 +635,13 @@ def check_completion_script_compatibility( # check that the installed completion script is compabile with this # completion server version - for requirement in parse_requirements(REQUIRED_SCRIPT_VERSION): - # NOTE: there's only one requirement but we have to iterate to get it - if installed_version not in requirement: - is_compatible = False - print( - f'The Cylc {completion_lang} script needs to be updated to' - ' work with this version of Cylc.', - file=sys.stderr, - ) + if installed_version not in SpecifierSet(REQUIRED_SCRIPT_VERSION): + is_compatible = False + print( + f'The Cylc {completion_lang} script needs to be updated to' + ' work with this version of Cylc.', + file=sys.stderr, + ) # check for completion script updates if installed_version < current_version: diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py index 4f13dc59496..46c8127585a 100644 --- a/cylc/flow/scripts/cylc.py +++ b/cylc/flow/scripts/cylc.py @@ -16,14 +16,55 @@ # along with this program. If not, see . """cylc main entry point""" -import argparse -from contextlib import contextmanager import os import sys + + +def pythonpath_manip(): + """Stop PYTHONPATH contaminating the Cylc Environment + + * Remove PYTHONPATH items from sys.path to prevent PYTHONPATH + contaminating the Cylc Environment. + * Add items from CYLC_PYTHONPATH to sys.path. + + See Also: + https://github.com/cylc/cylc-flow/issues/5124 + """ + if 'CYLC_PYTHONPATH' in os.environ: + for item in os.environ['CYLC_PYTHONPATH'].split(os.pathsep): + abspath = os.path.abspath(item) + sys.path.insert(0, abspath) + if 'PYTHONPATH' in os.environ: + for item in os.environ['PYTHONPATH'].split(os.pathsep): + abspath = os.path.abspath(item) + if abspath in sys.path: + sys.path.remove(abspath) + + +pythonpath_manip() + +if sys.version_info[:2] > (3, 11): + from importlib.metadata import ( + entry_points, + files, + ) +else: + # BACK COMPAT: importlib_metadata + # importlib.metadata was added in Python 3.8. The required interfaces + # were completed by 3.12. For lower versions we must use the + # importlib_metadata backport. + # FROM: Python 3.7 + # TO: Python: 3.12 + from importlib_metadata import ( + entry_points, + files, + ) + +import argparse +from contextlib import contextmanager from typing import Iterator, NoReturn, Optional, Tuple from ansimarkup import parse as cparse -import pkg_resources from cylc.flow import __version__, iter_entry_points from cylc.flow.option_parsers import ( @@ -281,9 +322,9 @@ def execute_cmd(cmd: str, *args: str) -> NoReturn: args: Command line arguments to pass to that command. """ - entry_point: pkg_resources.EntryPoint = COMMANDS[cmd] + entry_point = COMMANDS[cmd] try: - entry_point.resolve()(*args) + entry_point.load()(*args) except ModuleNotFoundError as exc: msg = handle_missing_dependency(entry_point, exc) print(msg, file=sys.stderr) @@ -376,7 +417,7 @@ def iter_commands() -> Iterator[Tuple[str, Optional[str], Optional[str]]]: """ for cmd, entry_point in sorted(COMMANDS.items()): try: - module = __import__(entry_point.module_name, fromlist=['']) + module = __import__(entry_point.module, fromlist=['']) except ModuleNotFoundError as exc: handle_missing_dependency(entry_point, exc) continue @@ -392,18 +433,10 @@ def print_id_help(): def print_license() -> None: - try: - from importlib.metadata import files - except ImportError: - # BACK COMPAT: importlib_metadata - # importlib.metadata was added in Python 3.8 - # FROM: Python 3.7 - # TO: Python: 3.8 - from importlib_metadata import files # type: ignore[no-redef] - license_file = next(filter( - lambda f: f.name == 'COPYING', files('cylc-flow') - )) - print(license_file.read_text()) + for file in files('cylc-flow') or []: + if file.name == 'COPYING': + print(file.read_text()) + return def print_command_list(commands=None, indent=0): @@ -442,54 +475,57 @@ def cli_version(long_fmt=False): """Wrapper for get_version.""" print(get_version(long_fmt)) if long_fmt: - print(list_plugins()) + print(cparse(list_plugins())) sys.exit(0) def list_plugins(): - entry_point_names = [ - entry_point_name - for entry_point_name - in pkg_resources.get_entry_map('cylc-flow').keys() - if entry_point_name.startswith('cylc.') - ] - - entry_point_groups = { - entry_point_name: [ - entry_point - for entry_point - in iter_entry_points(entry_point_name) - if not entry_point.module_name.startswith('cylc.flow') - ] - for entry_point_name in entry_point_names - } - - dists = { - entry_point.dist - for entry_points in entry_point_groups.values() - for entry_point in entry_points - } - - lines = [] - if dists: - lines.append('\nPlugins:') - maxlen1 = max(len(dist.project_name) for dist in dists) + 2 - maxlen2 = max(len(dist.version) for dist in dists) + 2 - for dist in dists: - lines.append( - f' {dist.project_name.ljust(maxlen1)}' - f' {dist.version.ljust(maxlen2)}' - f' {dist.module_path}' - ) - - lines.append('\nEntry Points:') - for entry_point_name, entry_points in entry_point_groups.items(): - if entry_points: - lines.append(f' {entry_point_name}:') - for entry_point in entry_points: - lines.append(f' {entry_point}') - - return '\n'.join(lines) + from cylc.flow.terminal import DIM, format_grid + # go through all Cylc entry points + _dists = set() + __entry_points = {} + for entry_point in entry_points(): + if ( + # all Cylc entry points are under the "cylc" namespace + entry_point.group.startswith('cylc.') + # don't list cylc-flow entry-points (i.e. built-ins) + and not entry_point.value.startswith('cylc.flow') + ): + _dists.add(entry_point.dist) + __entry_points.setdefault( + entry_point.group, + [], + ).append(entry_point) + + # list all the distriutions which provide Cylc entry points + _plugins = [] + for dist in _dists: + _plugins.append(( + '', + f'{dist.name}', + dist.version, + f'<{DIM}>{dist.locate_file("__init__.py").parent}', + )) + + # list all of the entry points by "group" (e.g. "cylc.command") + _entry_points = [] + for group, points in sorted(__entry_points.items()): + _entry_points.append((f' {group}:', '', '')) + for entry_point in points: + _entry_points.append(( + f' {entry_point.name}', + f'{entry_point.dist.name}', + f'<{DIM}>{entry_point.value}', + )) + + return '\n'.join(( + '\nPlugins:', + *format_grid(_plugins), + '\nEntry Points:', + *format_grid( + _entry_points + ), + )) @contextmanager @@ -661,7 +697,7 @@ def main(): def handle_missing_dependency( - entry_point: pkg_resources.EntryPoint, + entry_point, err: ModuleNotFoundError ) -> str: """Return a suitable error message for a missing optional dependency. @@ -673,12 +709,8 @@ def handle_missing_dependency( Re-raises the given ModuleNotFoundError if it is unexpected. """ - try: - # Check for missing optional dependencies - entry_point.require() - except pkg_resources.DistributionNotFound as exc: - # Confirmed missing optional dependencies - return f"cylc {entry_point.name}: {exc}" - else: - # Error not due to missing optional dependencies; this is unexpected - raise err + msg = f'"cylc {entry_point.name}" requires "{entry_point.dist.name}' + if entry_point.extras: + msg += f'[{",".join(entry_point.extras)}]' + msg += f'"\n\n{err.__class__.__name__}: {err}' + return msg diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py index 45010e4c714..be9583137e9 100755 --- a/cylc/flow/scripts/install.py +++ b/cylc/flow/scripts/install.py @@ -301,7 +301,7 @@ def install( 'cylc.pre_configure' ): try: - entry_point.resolve()(srcdir=source, opts=opts) + entry_point.load()(srcdir=source, opts=opts) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors @@ -329,7 +329,7 @@ def install( 'cylc.post_install' ): try: - entry_point.resolve()( + entry_point.load()( srcdir=source_dir, opts=opts, rundir=str(rundir) diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py index 951e6aa3643..cc06512988c 100755 --- a/cylc/flow/scripts/lint.py +++ b/cylc/flow/scripts/lint.py @@ -22,6 +22,8 @@ # NOTE: docstring needed for `cylc help all` output # (if editing check this still comes out as expected) +LINT_SECTIONS = ['cylc-lint', 'cylclint', 'cylc_lint'] + COP_DOC = """cylc lint [OPTIONS] ARGS Check .cylc and .rc files for code style, deprecated syntax and other issues. @@ -33,9 +35,15 @@ A non-zero return code will be returned if any issues are identified. This can be overridden by providing the "--exit-zero" flag. +""" -Configurations for Cylc lint can also be set in a pyproject.toml file. - +TOMLDOC = """ +pyproject.toml configuration:{} + [cylc-lint] # any of {} + ignore = ['S001', 'S002'] # List of rules to ignore + exclude = ['etc/foo.cylc'] # List of files to ignore + rulesets = ['style', '728'] # Sets default rulesets to check + max-line-length = 130 # Max line length for linting """ from colorama import Fore import functools @@ -58,7 +66,8 @@ loads as toml_loads, TOMLDecodeError, ) -from typing import Callable, Dict, Iterator, List, Union, TYPE_CHECKING +from typing import ( + TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Union) from cylc.flow import LOG from cylc.flow.exceptions import CylcError @@ -224,13 +233,44 @@ def check_for_obsolete_environment_variables(line: str) -> List[str]: return [i for i in OBSOLETE_ENV_VARS if i in line] +INDENTATION = re.compile(r'^(\s*)(.*)') + + +def check_indentation(line: str) -> bool: + """The key value pair is not indented 4*X spaces + + n.b. We test for trailing whitespace and incorrect section indenting + elsewhere + + Examples: + + >>> check_indentation('') + False + >>> check_indentation(' ') + False + >>> check_indentation(' [') + False + >>> check_indentation('baz') + False + >>> check_indentation(' qux') + False + >>> check_indentation(' foo') + True + >>> check_indentation(' bar') + True + """ + match = INDENTATION.findall(line)[0] + if not match[0] or not match[1] or match[1].startswith('['): + return False + return bool(len(match[0]) % 4 != 0) + + FUNCTION = 'function' STYLE_GUIDE = ( 'https://cylc.github.io/cylc-doc/stable/html/workflow-design-guide/' 'style-guide.html#' ) -URL_STUB = "https://cylc.github.io/cylc-doc/stable/html/7-to-8/" SECTION2 = r'\[\[\s*{}\s*\]\]' SECTION3 = r'\[\[\[\s*{}\s*\]\]\]' FILEGLOBS = ['*.rc', '*.cylc'] @@ -270,7 +310,6 @@ def check_for_obsolete_environment_variables(line: str) -> List[str]: # - short: A short description of the issue. # - url: A link to a fuller description. # - function: A function to use to run the check. -# - fallback: A second function(The first function might want to call this?) # - kwargs: We want to pass a set of common kwargs to the check function. # - evaluate commented lines: Run this check on commented lines. # - rst: An rst description, for use in the Cylc docs. @@ -398,23 +437,34 @@ def check_for_obsolete_environment_variables(line: str) -> List[str]: 'evaluate commented lines': True, FUNCTION: functools.partial( check_if_jinja2, - function=re.compile(r'(? List[str]: 'job-script-vars/index.html' ), FUNCTION: check_for_obsolete_environment_variables, - } + }, + 'U014': { + 'short': 'Use "isodatetime [ref]" instead of "rose date [-c]"', + 'rst': ( + 'For datetime operations in task scripts:\n\n' + ' * Use ``isodatetime`` instead of ``rose date``\n' + ' * Use ``isodatetime ref`` instead of ``rose date -c`` for ' + 'the current cycle point\n' + ), + 'url': ( + 'https://cylc.github.io/cylc-doc/stable/html/7-to-8/' + 'cheat-sheet.html#datetime-operations' + ), + FUNCTION: re.compile(r'rose +date').findall, + }, } RULESETS = ['728', 'style', 'all'] EXTRA_TOML_VALIDATION = { @@ -543,6 +607,31 @@ def check_for_obsolete_environment_variables(line: str) -> List[str]: } +def get_url(check_meta: Dict) -> str: + """Get URL from check data. + + If the URL doesn't start with http then prepend with address + of the 7-to-8 upgrade guide. + + Examples: + >>> get_url({'no': 'url key'}) + '' + >>> get_url({'url': ''}) + '' + >>> get_url({'url': 'https://www.h2g2.com/'}) + 'https://www.h2g2.com/' + >>> get_url({'url': 'cheat-sheet.html'}) + 'https://cylc.github.io/cylc-doc/stable/html/7-to-8/cheat-sheet.html' + """ + url = check_meta.get('url', '') + if url and not url.startswith('http'): + url = ( + "https://cylc.github.io/cylc-doc/stable/html/7-to-8/" + + check_meta['url'] + ) + return url + + def validate_toml_items(tomldata): """Check that all tomldata items are lists of strings @@ -592,7 +681,7 @@ def get_pyproject_toml(dir_): raise CylcError(f'pyproject.toml did not load: {exc}') if any( - i in loadeddata for i in ['cylc-lint', 'cylclint', 'cylc_lint'] + i in loadeddata for i in LINT_SECTIONS ): for key in keys: tomldata[key] = loadeddata.get('cylc-lint').get(key, []) @@ -602,10 +691,43 @@ def get_pyproject_toml(dir_): return tomldata -def merge_cli_with_tomldata( - clidata, tomldata, - override_cli_default_rules=False -): +def merge_cli_with_tomldata(target: Path, options: 'Values') -> Dict[str, Any]: + """Get a list of checks based on the checking options + + Args: + target: Location being linted, in which we might find a + pyproject.toml file. + options: Cli Options + + This has not been merged with merged with the logic in + _merge_cli_with_tomldata to keep the testing of file-system touching + and pure logic separate. + """ + ruleset_default = False + if options.linter == 'all': + options.linter = ['728', 'style'] + elif options.linter == '': + options.linter = ['728', 'style'] + ruleset_default = True + else: + options.linter = [options.linter] + tomlopts = get_pyproject_toml(target) + return _merge_cli_with_tomldata( + { + 'exclude': [], + 'ignore': options.ignores, + 'rulesets': options.linter + }, + tomlopts, + ruleset_default + ) + + +def _merge_cli_with_tomldata( + clidata: Dict[str, Any], + tomldata: Dict[str, Any], + override_cli_default_rules: bool = False +) -> Dict[str, Any]: """Merge options set by pyproject.toml with CLI options. rulesets: CLI should override toml. @@ -619,7 +741,7 @@ def merge_cli_with_tomldata( default, but only if we ask for it on the CLI. Examples: - >>> result = merge_cli_with_tomldata( + >>> result = _merge_cli_with_tomldata( ... {'rulesets': ['foo'], 'ignore': ['R101'], 'exclude': []}, ... {'rulesets': ['bar'], 'ignore': ['R100'], 'exclude': ['*.bk']}) >>> result['ignore'] @@ -908,10 +1030,7 @@ def lint( counter[check_meta['purpose']] += 1 if modify: # insert a command to help the user - if check_meta['url'].startswith('http'): - url = check_meta['url'] - else: - url = URL_STUB + check_meta['url'] + url = get_url(check_meta) yield ( f'# [{get_index_str(check_meta, index)}]: ' @@ -954,99 +1073,105 @@ def get_cylc_files( yield path -def get_reference_rst(checks): - """Print a reference for checks to be carried out. +REFERENCE_TEMPLATES = { + 'section heading': '\n{title}\n{underline}\n', + 'issue heading': { + 'text': '\n{check}:\n {summary}\n {url}\n\n', + 'rst': '\n`{check} <{url}>`_\n{underline}\n{summary}\n\n', + }, + 'auto gen message': ( + 'U998 and U999 represent automatically generated' + ' sets of deprecations and upgrades.' + ), +} + - Returns: - RST compatible text. +def get_reference(linter, output_type): + """Fill out a template with all the issues Cylc Lint looks for. """ + if linter in {'all', ''}: + rulesets = ['728', 'style'] + else: + rulesets = [linter] + checks = parse_checks(rulesets, reference=True) + + issue_heading_template = REFERENCE_TEMPLATES['issue heading'][output_type] output = '' current_checkset = '' for index, meta in checks.items(): # Check if the purpose has changed - if so create a new - # section title: + # section heading: if meta['purpose'] != current_checkset: current_checkset = meta['purpose'] title = CHECKS_DESC[meta["purpose"]] - output += f'\n{title}\n{"-" * len(title)}\n\n' + output += REFERENCE_TEMPLATES['section heading'].format( + title=title, underline="-" * len(title)) if current_checkset == 'A': - output += ( - '\n.. note::\n' - '\n U998 and U999 represent automatically generated ' - 'sets of deprecations and upgrades.\n\n' - ) + output += REFERENCE_TEMPLATES['auto gen message'] - if current_checkset == 'A': + # Fill a template with info about the issue. + if output_type == 'rst': summary = meta.get("rst", meta['short']) - output += '\n- ' + summary + elif output_type == 'text': + summary = meta.get("short").replace('``', '') + + if current_checkset == 'A': + # Condensed check summary for auto-generated lint items. + if output_type == 'rst': + output += '\n' + output += '\n* ' + summary else: - # Fill a template with info about the issue. - template = ( - '{check}\n^^^^\n{summary}\n\n' - ) - if meta['url'].startswith('http'): - url = meta['url'] - else: - url = URL_STUB + meta['url'] - summary = meta.get("rst", meta['short']) + template = issue_heading_template + url = get_url(meta) msg = template.format( + title=index, check=get_index_str(meta, index), summary=summary, url=url, + underline=( + len(get_index_str(meta, index)) + len(url) + 6 + ) * '^' ) output += msg output += '\n' return output -def get_reference_text(checks): - """Print a reference for checks to be carried out. - - Returns: - RST compatible text. +def target_version_check( + target: Path, + quiet: 'Values', + mergedopts: Dict[str, Any] +) -> List: """ - output = '' - current_checkset = '' - for index, meta in checks.items(): - # Check if the purpose has changed - if so create a new - # section title: - if meta['purpose'] != current_checkset: - current_checkset = meta['purpose'] - title = CHECKS_DESC[meta["purpose"]] - output += f'\n{title}\n{"-" * len(title)}\n\n' + Check whether target is an upgraded Cylc 8 workflow. - if current_checkset == 'A': - output += ( - 'U998 and U999 represent automatically generated' - ' sets of deprecations and upgrades.' - ) - # Fill a template with info about the issue. - if current_checkset == 'A': - summary = meta.get("rst", meta['short']).replace('``', '') - output += '\n* ' + summary - else: - template = ( - '{check}:\n {summary}\n\n' - ) - if meta['url'].startswith('http'): - url = meta['url'] - else: - url = URL_STUB + meta['url'] - msg = template.format( - title=index, - check=get_index_str(meta, index), - summary=meta['short'], - url=url, - ) - output += msg - output += '\n' - return output + If it isn't then we shouldn't run the 7-to-8 checks upon + it. + + If it isn't and the only ruleset requested by the user is '728' + we should exit with an error code unless the user has specifically + disabled thatr with --exit-zero. + """ + cylc8 = (target / 'flow.cylc').exists() + if not cylc8 and mergedopts['rulesets'] == ['728']: + LOG.error( + f'{target} not a Cylc 8 workflow: ' + 'Lint after renaming ' + '"suite.rc" to "flow.cylc"' + ) + sys.exit(not quiet) + elif not cylc8 and '728' in mergedopts['rulesets']: + check_names = mergedopts['rulesets'] + check_names.remove('728') + else: + check_names = mergedopts['rulesets'] + return check_names def get_option_parser() -> COP: parser = COP( - COP_DOC, + COP_DOC + TOMLDOC.format('', str(LINT_SECTIONS)), argdoc=[ COP.optional(WORKFLOW_ID_OR_PATH_ARG_DOC) ], @@ -1088,7 +1213,7 @@ def get_option_parser() -> COP: default=[], dest='ignores', metavar="CODE", - choices=tuple(STYLE_CHECKS) + choices=list(STYLE_CHECKS.keys()) + [LINE_LEN_NO] ) parser.add_option( '--exit-zero', @@ -1104,72 +1229,37 @@ def get_option_parser() -> COP: @cli_function(get_option_parser) def main(parser: COP, options: 'Values', target=None) -> None: if options.ref_mode: - if options.linter in {'all', ''}: - rulesets = ['728', 'style'] - else: - rulesets = [options.linter] - print(get_reference_text(parse_checks(rulesets, reference=True))) + print(get_reference(options.linter, 'text')) sys.exit(0) - # If target not given assume we are looking at PWD + # If target not given assume we are looking at PWD: if target is None: target = str(Path.cwd()) - # make sure the target is a src/run directories + # make sure the target is a src/run directory: _, _, target = parse_id( target, src=True, constraint='workflows', ) - # Get a list of checks bas ed on the checking options: - # Allow us to check any number of folders at once + # We want target to be the containing folder, not the flow.cylc + # file identified by parse_id: target = target.parent - ruleset_default = False - if options.linter == 'all': - options.linter = ['728', 'style'] - elif options.linter == '': - options.linter = ['728', 'style'] - ruleset_default = True - else: - options.linter = [options.linter] - tomlopts = get_pyproject_toml(target) - mergedopts = merge_cli_with_tomldata( - { - 'exclude': [], - 'ignore': options.ignores, - 'rulesets': options.linter - }, - tomlopts, - ruleset_default - ) - # Check whether target is an upgraded Cylc 8 workflow. - # If it isn't then we shouldn't run the 7-to-8 checks upon - # it: - cylc8 = (target / 'flow.cylc').exists() - if not cylc8 and mergedopts['rulesets'] == ['728']: - LOG.error( - f'{target} not a Cylc 8 workflow: ' - 'Lint after renaming ' - '"suite.rc" to "flow.cylc"' - ) - # Exit with an error code if --exit-zero was not set. - # Return codes: sys.exit(True) == 1, sys.exit(False) == 0 - sys.exit(not options.exit_zero) - elif not cylc8 and '728' in mergedopts['rulesets']: - check_names = mergedopts['rulesets'] - check_names.remove('728') - else: - check_names = mergedopts['rulesets'] + mergedopts = merge_cli_with_tomldata(target, options) + + check_names = target_version_check( + target=target, quiet=options.exit_zero, mergedopts=mergedopts) - # Check each file: + # Get the checks object. checks = parse_checks( check_names, ignores=mergedopts['ignore'], max_line_len=mergedopts['max-line-length'] ) + # Check each file matching a pattern: counter: Dict[str, int] = {} for file in get_cylc_files(target, mergedopts['exclude']): LOG.debug(f'Checking {file}') @@ -1206,4 +1296,4 @@ def main(parser: COP, options: 'Values', target=None) -> None: # NOTE: use += so that this works with __import__ # (docstring needed for `cylc help all` output) -__doc__ += get_reference_rst(parse_checks(['728', 'style'], reference=True)) +__doc__ += get_reference('all', 'rst') diff --git a/cylc/flow/scripts/reinstall.py b/cylc/flow/scripts/reinstall.py index c3b48b23c74..a2acbc7f771 100644 --- a/cylc/flow/scripts/reinstall.py +++ b/cylc/flow/scripts/reinstall.py @@ -71,6 +71,7 @@ from pathlib import Path import sys from typing import Optional, TYPE_CHECKING, List, Callable +from functools import partial from ansimarkup import parse as cparse @@ -83,12 +84,13 @@ from cylc.flow.install import ( reinstall_workflow, ) -from cylc.flow.id_cli import parse_id +from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( CylcOptionParser as COP, OptionSettings, - WORKFLOW_ID_ARG_DOC, + ID_MULTI_ARG_DOC ) + from cylc.flow.pathutil import get_workflow_run_dir from cylc.flow.workflow_files import ( get_workflow_source_dir, @@ -101,7 +103,6 @@ _input = input # to enable testing - REINSTALL_CYLC_ROSE_OPTIONS = [ OptionSettings( ['--clear-rose-install-options'], @@ -127,7 +128,12 @@ def get_option_parser() -> COP: parser = COP( - __doc__, comms=True, argdoc=[WORKFLOW_ID_ARG_DOC] + __doc__, + comms=True, + multiworkflow=True, + argdoc=[ + ID_MULTI_ARG_DOC, + ], ) try: @@ -149,27 +155,28 @@ def get_option_parser() -> COP: def main( _parser: COP, opts: 'Values', - args: Optional[str] = None + *ids: str ) -> None: """CLI wrapper.""" - reinstall_cli(opts, args) + call_multi( + partial(reinstall_cli, opts), + *ids, + constraint='workflows', + report=lambda x: print('Done') + ) -def reinstall_cli( +async def reinstall_cli( opts: 'Values', - args: Optional[str] = None, - print_reload_tip: bool = True, + workflow_id, + *tokens_list, + print_reload_tip: bool = True ) -> bool: """Implement cylc reinstall. This is the bit which contains all the CLI logic. """ run_dir: Optional[Path] - workflow_id: str - workflow_id, *_ = parse_id( - args, - constraint='workflows', - ) run_dir = Path(get_workflow_run_dir(workflow_id)) if not run_dir.is_dir(): raise WorkflowFilesError( @@ -338,7 +345,7 @@ def pre_configure(opts: 'Values', src_dir: Path) -> None: 'cylc.pre_configure' ): try: - entry_point.resolve()(srcdir=src_dir, opts=opts) + entry_point.load()(srcdir=src_dir, opts=opts) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors @@ -355,7 +362,7 @@ def post_install(opts: 'Values', src_dir: Path, run_dir: Path) -> None: 'cylc.post_install' ): try: - entry_point.resolve()( + entry_point.load()( srcdir=src_dir, opts=opts, rundir=str(run_dir) diff --git a/cylc/flow/scripts/report_timings.py b/cylc/flow/scripts/report_timings.py index 8d05ed2748d..b6d796ad732 100755 --- a/cylc/flow/scripts/report_timings.py +++ b/cylc/flow/scripts/report_timings.py @@ -54,6 +54,8 @@ import sys from typing import TYPE_CHECKING + +from cylc.flow import LOG from cylc.flow.exceptions import CylcError from cylc.flow.id_cli import parse_id from cylc.flow.option_parsers import ( @@ -123,6 +125,12 @@ def main(parser: COP, options: 'Values', workflow_id: str) -> None: constraint='workflows', ) + LOG.warning( + "cylc report-timings is deprecated." + " The analysis view in the GUI provides" + " similar functionality." + ) + output_options = [ options.show_raw, options.show_summary, options.html_summary ] @@ -246,7 +254,10 @@ def _check_imports(self): try: import pandas except ImportError: - raise CylcError('Cannot import pandas - summary unavailable.') + raise CylcError( + 'Cannot import pandas - summary unavailable.' + ' try: pip install cylc-flow[report-timings]' + ) else: del pandas diff --git a/cylc/flow/scripts/show.py b/cylc/flow/scripts/show.py index 5a36eaf8e0d..dae42f637b8 100755 --- a/cylc/flow/scripts/show.py +++ b/cylc/flow/scripts/show.py @@ -37,12 +37,16 @@ """ import asyncio +import re import json import sys -from typing import Dict, TYPE_CHECKING +from typing import Any, Dict, TYPE_CHECKING from ansimarkup import ansiprint +from metomi.isodatetime.data import ( + get_timepoint_from_seconds_since_unix_epoch as seconds2point) + from cylc.flow.exceptions import InputError from cylc.flow.id import Tokens from cylc.flow.id_cli import parse_ids @@ -337,9 +341,10 @@ async def prereqs_and_outputs_query( f'{ext_trig["label"]} ... {state}', state) for xtrig in t_proxy['xtriggers']: + label = get_wallclock_label(xtrig) or xtrig['id'] state = xtrig['satisfied'] print_msg_state( - f'xtrigger "{xtrig["label"]} = {xtrig["id"]}"', + f'xtrigger "{xtrig["label"]} = {label}"', state) if not results['taskProxies']: ansiprint( @@ -349,6 +354,32 @@ async def prereqs_and_outputs_query( return 0 +def get_wallclock_label(xtrig: Dict[str, Any]) -> str: + """Return a label for an xtrigger if it is a wall_clock trigger. + + Returns: + A label or False. + + Examples: + >>> this = get_wallclock_label + + >>> this({'id': 'wall_clock(trigger_time=0)'}) + 'wall_clock(trigger_time=1970-01-01T00:00:00Z)' + + >>> this({'id': 'wall_clock(trigger_time=440143843)'}) + 'wall_clock(trigger_time=1983-12-13T06:10:43Z)' + + """ + wallclock_trigger = re.findall( + r'wall_clock\(trigger_time=(.*)\)', xtrig['id']) + if wallclock_trigger: + return ( + 'wall_clock(trigger_time=' + f'{str(seconds2point(wallclock_trigger[0], True))})' + ) + return '' + + async def task_meta_query( workflow_id, task_names, diff --git a/cylc/flow/scripts/tui.py b/cylc/flow/scripts/tui.py index 86970052b46..f8f0879a1e6 100644 --- a/cylc/flow/scripts/tui.py +++ b/cylc/flow/scripts/tui.py @@ -15,34 +15,35 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""cylc tui WORKFLOW +"""cylc tui [WORKFLOW] View and control running workflows in the terminal. (Tui = Terminal User Interface) -WARNING: Tui is experimental and may break with large flows. -An upcoming change to the way Tui receives data from the scheduler will make it -much more efficient in the future. +Tui allows you to monitor and interact with workflows in a manner similar +to the GUI. + +Press "h" whilst running Tui to bring up the help screen, use the arrow +keys to navigage. + """ -# TODO: remove this warning once Tui is delta-driven -# https://github.com/cylc/cylc-flow/issues/3527 +from getpass import getuser from textwrap import indent -from typing import TYPE_CHECKING -from urwid import html_fragment +from typing import TYPE_CHECKING, Optional +from cylc.flow.id import Tokens from cylc.flow.id_cli import parse_id from cylc.flow.option_parsers import ( - WORKFLOW_ID_ARG_DOC, + OPT_WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function from cylc.flow.tui import TUI +from cylc.flow.tui.util import suppress_logging from cylc.flow.tui.app import ( TuiApp, - TREE_EXPAND_DEPTH - # ^ a nasty solution ) if TYPE_CHECKING: @@ -55,57 +56,25 @@ def get_option_parser() -> COP: parser = COP( __doc__, - argdoc=[WORKFLOW_ID_ARG_DOC], + argdoc=[OPT_WORKFLOW_ID_ARG_DOC], # auto_add=False, NOTE: at present auto_add can not be turned off color=False ) - parser.add_option( - '--display', - help=( - 'Specify the display technology to use.' - ' "raw" for interactive in-terminal display.' - ' "html" for non-interactive html output.' - ), - action='store', - choices=['raw', 'html'], - default='raw', - ) - parser.add_option( - '--v-term-size', - help=( - 'The virtual terminal size for non-interactive' - '--display options.' - ), - action='store', - default='80,24' - ) - return parser @cli_function(get_option_parser) -def main(_, options: 'Values', workflow_id: str) -> None: - workflow_id, *_ = parse_id( - workflow_id, - constraint='workflows', - ) - screen = None - if options.display == 'html': - TREE_EXPAND_DEPTH[0] = -1 # expand tree fully - screen = html_fragment.HtmlGenerator() - screen.set_terminal_properties(256) - screen.register_palette(TuiApp.palette) - html_fragment.screenshot_init( - [tuple(map(int, options.v_term_size.split(',')))], - [] +def main(_, options: 'Values', workflow_id: Optional[str] = None) -> None: + # get workflow ID if specified + if workflow_id: + workflow_id, *_ = parse_id( + workflow_id, + constraint='workflows', ) + tokens = Tokens(workflow_id) + workflow_id = tokens.duplicate(user=getuser()).id - try: - TuiApp(workflow_id, screen=screen).main() - - if options.display == 'html': - for fragment in html_fragment.screenshot_collect(): - print(fragment) - except KeyboardInterrupt: + # start Tui + with suppress_logging(), TuiApp().main(workflow_id): pass diff --git a/cylc/flow/scripts/validate_reinstall.py b/cylc/flow/scripts/validate_reinstall.py index 7a4472901ef..3fb665cf999 100644 --- a/cylc/flow/scripts/validate_reinstall.py +++ b/cylc/flow/scripts/validate_reinstall.py @@ -50,7 +50,7 @@ ContactFileExists, CylcError, ) -from cylc.flow.id_cli import parse_id +from cylc.flow.id_cli import parse_id_async from cylc.flow.loggingutil import set_timestamps from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, @@ -62,7 +62,7 @@ from cylc.flow.scheduler_cli import PLAY_OPTIONS, scheduler_cli from cylc.flow.scripts.validate import ( VALIDATE_OPTIONS, - _main as cylc_validate + wrapped_main as cylc_validate ) from cylc.flow.scripts.reinstall import ( REINSTALL_CYLC_ROSE_OPTIONS, @@ -70,11 +70,12 @@ reinstall_cli as cylc_reinstall, ) from cylc.flow.scripts.reload import ( - reload_cli as cylc_reload + run as cylc_reload ) from cylc.flow.terminal import cli_function from cylc.flow.workflow_files import detect_old_contact_file +import asyncio CYLC_ROSE_OPTIONS = COP.get_cylc_rose_options() VR_OPTIONS = combine_options( @@ -124,16 +125,16 @@ def check_tvars_and_workflow_stopped( @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str): - sys.exit(vro_cli(parser, options, workflow_id)) + sys.exit(asyncio.run(vr_cli(parser, options, workflow_id))) -def vro_cli(parser: COP, options: 'Values', workflow_id: str): +async def vr_cli(parser: COP, options: 'Values', workflow_id: str): """Run Cylc (re)validate - reinstall - reload in sequence.""" # Attempt to work out whether the workflow is running. # We are trying to avoid reinstalling then subsequently being # unable to play or reload because we cannot identify workflow state. unparsed_wid = workflow_id - workflow_id, *_ = parse_id( + workflow_id, *_ = await parse_id_async( workflow_id, constraint='workflows', ) @@ -166,10 +167,14 @@ def vro_cli(parser: COP, options: 'Values', workflow_id: str): # Force on the against_source option: options.against_source = True # Make validate check against source. log_subcommand('validate --against-source', workflow_id) - cylc_validate(parser, options, workflow_id) + await cylc_validate(parser, options, workflow_id) log_subcommand('reinstall', workflow_id) - reinstall_ok = cylc_reinstall(options, workflow_id, print_reload_tip=False) + reinstall_ok = await cylc_reinstall( + options, workflow_id, + [], + print_reload_tip=False + ) if not reinstall_ok: LOG.warning( 'No changes to source: No reinstall or' @@ -180,7 +185,7 @@ def vro_cli(parser: COP, options: 'Values', workflow_id: str): # Run reload if workflow is running or paused: if workflow_running: log_subcommand('reload', workflow_id) - cylc_reload(options, workflow_id) + await cylc_reload(options, workflow_id) # run play anyway, to play a stopped workflow: else: @@ -197,4 +202,4 @@ def vro_cli(parser: COP, options: 'Values', workflow_id: str): source='', # Intentionally blank ) log_subcommand(*sys.argv[1:]) - scheduler_cli(options, workflow_id) + await scheduler_cli(options, workflow_id, parse_workflow_id=False) diff --git a/cylc/flow/simulation.py b/cylc/flow/simulation.py new file mode 100644 index 00000000000..15314f8e3e7 --- /dev/null +++ b/cylc/flow/simulation.py @@ -0,0 +1,219 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Utilities supporting simulation and skip modes +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from time import time + +from cylc.flow.cycling.loader import get_point +from cylc.flow.network.resolvers import TaskMsg +from cylc.flow.platforms import FORBIDDEN_WITH_PLATFORM +from cylc.flow.task_state import ( + TASK_STATUS_RUNNING, + TASK_STATUS_FAILED, + TASK_STATUS_SUCCEEDED, +) +from cylc.flow.wallclock import get_current_time_string + +from metomi.isodatetime.parsers import DurationParser + +if TYPE_CHECKING: + from queue import Queue + from cylc.flow.cycling import PointBase + from cylc.flow.task_proxy import TaskProxy + + +def configure_sim_modes(taskdefs, sim_mode): + """Adjust task defs for simulation and dummy mode. + + """ + dummy_mode = bool(sim_mode == 'dummy') + + for tdef in taskdefs: + # Compute simulated run time by scaling the execution limit. + rtc = tdef.rtconfig + sleep_sec = get_simulated_run_len(rtc) + + rtc['execution time limit'] = ( + sleep_sec + DurationParser().parse(str( + rtc['simulation']['time limit buffer'])).get_seconds() + ) + + rtc['simulation']['simulated run length'] = sleep_sec + rtc['submission retry delays'] = [1] + + # Generate dummy scripting. + rtc['init-script'] = "" + rtc['env-script'] = "" + rtc['pre-script'] = "" + rtc['post-script'] = "" + rtc['script'] = build_dummy_script( + rtc, sleep_sec) if dummy_mode else "" + + disable_platforms(rtc) + + # Disable environment, in case it depends on env-script. + rtc['environment'] = {} + + rtc["simulation"][ + "fail cycle points" + ] = parse_fail_cycle_points( + rtc["simulation"]["fail cycle points"] + ) + + +def get_simulated_run_len(rtc: Dict[str, Any]) -> int: + """Get simulated run time. + + rtc = run time config + """ + limit = rtc['execution time limit'] + speedup = rtc['simulation']['speedup factor'] + if limit and speedup: + sleep_sec = (DurationParser().parse( + str(limit)).get_seconds() / speedup) + else: + sleep_sec = DurationParser().parse( + str(rtc['simulation']['default run length']) + ).get_seconds() + + return sleep_sec + + +def build_dummy_script(rtc: Dict[str, Any], sleep_sec: int) -> str: + """Create fake scripting for dummy mode. + + This is for Dummy mode only. + """ + script = "sleep %d" % sleep_sec + # Dummy message outputs. + for msg in rtc['outputs'].values(): + script += "\ncylc message '%s'" % msg + if rtc['simulation']['fail try 1 only']: + arg1 = "true" + else: + arg1 = "false" + arg2 = " ".join(rtc['simulation']['fail cycle points']) + script += "\ncylc__job__dummy_result %s %s || exit 1" % (arg1, arg2) + return script + + +def disable_platforms( + rtc: Dict[str, Any] +) -> None: + """Force platform = localhost + + Remove legacy sections [job] and [remote], which would conflict + with setting platforms. + + This can be simplified when support for the FORBIDDEN_WITH_PLATFORM + configurations is dropped. + """ + for section, keys in FORBIDDEN_WITH_PLATFORM.items(): + if section in rtc: + for key in keys: + if key in rtc[section]: + rtc[section][key] = None + rtc['platform'] = 'localhost' + + +def parse_fail_cycle_points( + f_pts_orig: List[str] +) -> 'Union[None, List[PointBase]]': + """Parse `[simulation][fail cycle points]`. + + - None for "fail all points". + - Else a list of cycle point objects. + + Examples: + >>> this = parse_fail_cycle_points + >>> this(['all']) is None + True + >>> this([]) + [] + """ + f_pts: 'Optional[List[PointBase]]' + if 'all' in f_pts_orig: + f_pts = None + else: + f_pts = [] + for point_str in f_pts_orig: + f_pts.append(get_point(point_str).standardise()) + return f_pts + + +def sim_time_check( + message_queue: 'Queue[TaskMsg]', itasks: 'List[TaskProxy]' +) -> bool: + """Check if sim tasks have been "running" for as long as required. + + If they have change the task state. + + Returns: + True if _any_ simulated task state has changed. + """ + sim_task_state_changed = False + now = time() + for itask in itasks: + if itask.state.status != TASK_STATUS_RUNNING: + continue + # Started time is not set on restart + if itask.summary['started_time'] is None: + itask.summary['started_time'] = now + timeout = ( + itask.summary['started_time'] + + itask.tdef.rtconfig['simulation']['simulated run length'] + ) + if now > timeout: + job_d = itask.tokens.duplicate(job=str(itask.submit_num)) + now_str = get_current_time_string() + if sim_task_failed( + itask.tdef.rtconfig['simulation'], + itask.point, + itask.get_try_num() + ): + message_queue.put( + TaskMsg(job_d, now_str, 'CRITICAL', TASK_STATUS_FAILED) + ) + else: + # Simulate message outputs. + for msg in itask.tdef.rtconfig['outputs'].values(): + message_queue.put( + TaskMsg(job_d, now_str, 'DEBUG', msg) + ) + message_queue.put( + TaskMsg(job_d, now_str, 'DEBUG', TASK_STATUS_SUCCEEDED) + ) + sim_task_state_changed = True + return sim_task_state_changed + + +def sim_task_failed( + sim_conf: Dict[str, Any], + point: 'PointBase', + try_num: int, +) -> bool: + """Encapsulate logic for deciding whether a sim task has failed. + + Allows Unit testing. + """ + return ( + sim_conf['fail cycle points'] is None # i.e. "all" + or point in sim_conf['fail cycle points'] + ) and ( + try_num == 1 or not sim_conf['fail try 1 only'] + ) diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py index d4d9130ee27..ec5e467b8aa 100644 --- a/cylc/flow/task_events_mgr.py +++ b/cylc/flow/task_events_mgr.py @@ -575,6 +575,7 @@ def process_message( True: if polling is required to confirm a reversal of status. """ + # Log messages if event_time is None: event_time = get_current_time_string() @@ -617,7 +618,6 @@ def process_message( self.setup_event_handlers( itask, self.EVENT_STARTED, f'job {self.EVENT_STARTED}') self.spawn_func(itask, TASK_OUTPUT_STARTED) - if message == self.EVENT_STARTED: if ( flag == self.FLAG_RECEIVED @@ -777,24 +777,23 @@ def _process_message_check( return False if ( - itask.state(TASK_STATUS_WAITING) - and + itask.state(TASK_STATUS_WAITING) + and itask.tdef.run_mode == 'live' # Polling in live mode only. + and ( ( - ( - # task has a submit-retry lined up - TimerFlags.SUBMISSION_RETRY in itask.try_timers - and itask.try_timers[ - TimerFlags.SUBMISSION_RETRY].num > 0 - ) - or - ( - # task has an execution-retry lined up - TimerFlags.EXECUTION_RETRY in itask.try_timers - and itask.try_timers[ - TimerFlags.EXECUTION_RETRY].num > 0 - ) + # task has a submit-retry lined up + TimerFlags.SUBMISSION_RETRY in itask.try_timers + and itask.try_timers[ + TimerFlags.SUBMISSION_RETRY].num > 0 ) - + or + ( + # task has an execution-retry lined up + TimerFlags.EXECUTION_RETRY in itask.try_timers + and itask.try_timers[ + TimerFlags.EXECUTION_RETRY].num > 0 + ) + ) ): # Ignore messages if task has a retry lined up # (caused by polling overlapping with task failure) diff --git a/cylc/flow/task_job_mgr.py b/cylc/flow/task_job_mgr.py index 412ba23b97a..d6bc59fb255 100644 --- a/cylc/flow/task_job_mgr.py +++ b/cylc/flow/task_job_mgr.py @@ -260,12 +260,13 @@ def submit_task_jobs(self, workflow, itasks, curve_auth, Return (list): list of tasks that attempted submission. """ - if is_simulation: return self._simulation_submit_task_jobs(itasks, workflow) + # Prepare tasks for job submission prepared_tasks, bad_tasks = self.prep_submit_task_jobs( workflow, itasks) + # Reset consumed host selection results self.task_remote_mgr.subshell_eval_reset() @@ -999,16 +1000,17 @@ def _simulation_submit_task_jobs(self, itasks, workflow): itask.waiting_on_job_prep = False itask.submit_num += 1 self._set_retry_timers(itask) + itask.platform = {'name': 'SIMULATION'} itask.summary['job_runner_name'] = 'SIMULATION' itask.summary[self.KEY_EXECUTE_TIME_LIMIT] = ( - itask.tdef.rtconfig['job']['simulated run length'] + itask.tdef.rtconfig['simulation']['simulated run length'] ) itask.jobs.append( self.get_simulation_job_conf(itask, workflow) ) self.task_events_mgr.process_message( - itask, INFO, TASK_OUTPUT_SUBMITTED + itask, INFO, TASK_OUTPUT_SUBMITTED, ) return itasks diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py index 2e8d8a634ac..ac0e9f44e31 100644 --- a/cylc/flow/task_pool.py +++ b/cylc/flow/task_pool.py @@ -40,7 +40,6 @@ from cylc.flow.id import Tokens, detokenise from cylc.flow.id_cli import contains_fnmatch from cylc.flow.id_match import filter_ids -from cylc.flow.network.resolvers import TaskMsg from cylc.flow.workflow_status import StopMode from cylc.flow.task_action_timer import TaskActionTimer, TimerFlags from cylc.flow.task_events_mgr import ( @@ -74,7 +73,6 @@ from cylc.flow.flow_mgr import FLOW_ALL, FLOW_NONE, FLOW_NEW if TYPE_CHECKING: - from queue import Queue from cylc.flow.config import WorkflowConfig from cylc.flow.cycling import IntervalBase, PointBase from cylc.flow.data_store_mgr import DataStoreMgr @@ -842,13 +840,14 @@ def get_tasks_by_point(self) -> 'Dict[PointBase, List[TaskProxy]]': return point_itasks - def get_task(self, point, name): + def get_task(self, point, name) -> Optional[TaskProxy]: """Retrieve a task from the pool.""" rel_id = f'{point}/{name}' for pool in (self.main_pool, self.hidden_pool): tasks = pool.get(point) if tasks and rel_id in tasks: return tasks[rel_id] + return None def _get_hidden_task_by_id(self, id_: str) -> Optional[TaskProxy]: """Return runahead pool task by ID if it exists, or None.""" @@ -1756,45 +1755,6 @@ def force_trigger_tasks( return len(unmatched) - def sim_time_check(self, message_queue: 'Queue[TaskMsg]') -> bool: - """Simulation mode: simulate task run times and set states.""" - if not self.config.run_mode('simulation'): - return False - sim_task_state_changed = False - now = time() - for itask in self.get_tasks(): - if itask.state.status != TASK_STATUS_RUNNING: - continue - # Started time is not set on restart - if itask.summary['started_time'] is None: - itask.summary['started_time'] = now - timeout = (itask.summary['started_time'] + - itask.tdef.rtconfig['job']['simulated run length']) - if now > timeout: - conf = itask.tdef.rtconfig['simulation'] - job_d = itask.tokens.duplicate(job=str(itask.submit_num)) - now_str = get_current_time_string() - if ( - conf['fail cycle points'] is None # i.e. "all" - or itask.point in conf['fail cycle points'] - ) and ( - itask.get_try_num() == 1 or not conf['fail try 1 only'] - ): - message_queue.put( - TaskMsg(job_d, now_str, 'CRITICAL', TASK_STATUS_FAILED) - ) - else: - # Simulate message outputs. - for msg in itask.tdef.rtconfig['outputs'].values(): - message_queue.put( - TaskMsg(job_d, now_str, 'DEBUG', msg) - ) - message_queue.put( - TaskMsg(job_d, now_str, 'DEBUG', TASK_STATUS_SUCCEEDED) - ) - sim_task_state_changed = True - return sim_task_state_changed - def set_expired_tasks(self): res = False for itask in self.get_tasks(): diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py index 9d51794ec6c..3db5c731ec7 100644 --- a/cylc/flow/task_proxy.py +++ b/cylc/flow/task_proxy.py @@ -38,19 +38,19 @@ ) if TYPE_CHECKING: + from cylc.flow.id import Tokens from cylc.flow.cycling import PointBase from cylc.flow.task_action_timer import TaskActionTimer from cylc.flow.taskdef import TaskDef - from cylc.flow.id import Tokens class TaskProxy: """Represent an instance of a cycling task in a running workflow. Attributes: - .clock_trigger_time: - Clock trigger time in seconds since epoch. - (Used for wall_clock xtrigger). + .clock_trigger_times: + Memoization of clock trigger times (Used for wall_clock xtrigger): + {offset string: seconds from epoch} .expire_time: Time in seconds since epoch when this task is considered expired. .identity: @@ -152,7 +152,7 @@ class TaskProxy: # Memory optimization - constrain possible attributes to this list. __slots__ = [ - 'clock_trigger_time', + 'clock_trigger_times', 'expire_time', 'identity', 'is_late', @@ -244,7 +244,7 @@ def __init__( self.try_timers: Dict[str, 'TaskActionTimer'] = {} self.non_unique_events = Counter() # type: ignore # TODO: figure out - self.clock_trigger_time: Optional[float] = None + self.clock_trigger_times: Dict[str, int] = {} self.expire_time: Optional[float] = None self.late_time: Optional[float] = None self.is_late = is_late @@ -253,7 +253,10 @@ def __init__( self.state = TaskState(tdef, self.point, status, is_held) # Determine graph children of this task (for spawning). - self.graph_children = generate_graph_children(tdef, self.point) + if data_mode: + self.graph_children = {} + else: + self.graph_children = generate_graph_children(tdef, self.point) def __repr__(self) -> str: return f"<{self.__class__.__name__} '{self.tokens}'>" @@ -352,25 +355,31 @@ def get_point_as_seconds(self): self.point_as_seconds += utc_offset_in_seconds return self.point_as_seconds - def get_clock_trigger_time(self, offset_str): - """Compute, cache, and return trigger time relative to cycle point. + def get_clock_trigger_time( + self, + point: 'PointBase', offset_str: Optional[str] = None + ) -> int: + """Compute, cache and return trigger time relative to cycle point. Args: - offset_str: ISO8601Interval string, e.g. "PT2M". - Can be None for zero offset. + point: Task's cycle point. + offset_str: ISO8601 interval string, e.g. "PT2M". + Can be None for zero offset. Returns: Absolute trigger time in seconds since Unix epoch. """ - if self.clock_trigger_time is None: - if offset_str is None: - trigger_time = self.point + offset_str = offset_str if offset_str else 'P0Y' + if offset_str not in self.clock_trigger_times: + if offset_str == 'P0Y': + trigger_time = point else: - trigger_time = self.point + ISO8601Interval(offset_str) - self.clock_trigger_time = int( - point_parse(str(trigger_time)).seconds_since_unix_epoch - ) - return self.clock_trigger_time + trigger_time = point + ISO8601Interval(offset_str) + + offset = int( + point_parse(str(trigger_time)).seconds_since_unix_epoch) + self.clock_trigger_times[offset_str] = offset + return self.clock_trigger_times[offset_str] def get_try_num(self): """Return the number of automatic tries (try number).""" diff --git a/cylc/flow/task_queues/independent.py b/cylc/flow/task_queues/independent.py index d0e82995898..185edee4f2b 100644 --- a/cylc/flow/task_queues/independent.py +++ b/cylc/flow/task_queues/independent.py @@ -18,7 +18,7 @@ from collections import deque from contextlib import suppress -from typing import List, Set, Dict, Counter, Any, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Set, Dict, Counter, Any from cylc.flow.task_queues import TaskQueueManagerBase @@ -40,11 +40,11 @@ def push_task(self, itask: 'TaskProxy') -> None: if itask.tdef.name in self.members: self.deque.appendleft(itask) - def release(self, active: Counter[str]) -> 'List[TaskProxy]': + def release(self, active: Counter[str]) -> List['TaskProxy']: """Release tasks if below the active limit.""" # The "active" argument counts active tasks by name. - released: 'List[TaskProxy]' = [] - held: 'List[TaskProxy]' = [] + released: List['TaskProxy'] = [] + held: List['TaskProxy'] = [] n_active: int = 0 for mem in self.members: n_active += active[mem] @@ -113,20 +113,20 @@ def __init__(self, config["limit"], config["members"] ) - self.force_released: 'Set[TaskProxy]' = set() + self.force_released: Set['TaskProxy'] = set() def push_task(self, itask: 'TaskProxy') -> None: """Push a task to the appropriate queue.""" for queue in self.queues.values(): queue.push_task(itask) - def release_tasks(self, active: Counter[str]) -> 'List[TaskProxy]': + def release_tasks(self, active: Counter[str]) -> List['TaskProxy']: """Release tasks up to the queue limits.""" - released: 'List[TaskProxy]' = [] + released: List['TaskProxy'] = [] for queue in self.queues.values(): released += queue.release(active) if self.force_released: - released += list(self.force_released) + released.extend(self.force_released) self.force_released = set() return released diff --git a/cylc/flow/terminal.py b/cylc/flow/terminal.py index bc45f6d65d3..e848691252e 100644 --- a/cylc/flow/terminal.py +++ b/cylc/flow/terminal.py @@ -94,6 +94,48 @@ def print_contents(contents, padding=5, char='.', indent=0): print(f'{indent} {" " * title_width}{" " * padding}{line}') +def format_grid(rows, gutter=2): + """Format gridded text. + + This takes a 2D table of text and formats it to the maximum width of each + column and adds a bit of space between them. + + Args: + rows: + 2D list containing the text to format. + gutter: + The width of the gutter between columns. + + Examples: + >>> format_grid([ + ... ['a', 'b', 'ccccc'], + ... ['ddddd', 'e', 'f'], + ... ]) + ['a b ccccc ', + 'ddddd e f '] + + >>> format_grid([]) + [] + + """ + if not rows: + return rows + templ = [ + '{col:%d}' % (max( + len(row[ind]) + for row in rows + ) + gutter) + for ind in range(len(rows[0])) + ] + lines = [] + for row in rows: + ret = '' + for ind, col in enumerate(row): + ret += templ[ind].format(col=col) + lines.append(ret) + return lines + + def supports_color(): """Determine if running in a terminal which supports color. diff --git a/cylc/flow/tui/__init__.py b/cylc/flow/tui/__init__.py index 92e3bce0268..6beaeae059e 100644 --- a/cylc/flow/tui/__init__.py +++ b/cylc/flow/tui/__init__.py @@ -106,12 +106,26 @@ class Bindings: + """Represets key bindings for the Tui app.""" def __init__(self): self.bindings = [] self.groups = {} def bind(self, keys, group, desc, callback): + """Register a key binding. + + Args: + keys: + The keys to bind. + group: + The group to which this binding should belong. + desc: + Description for this binding, used to generate help. + callback: + The thing to call when this binding is pressed. + + """ if group not in self.groups: raise ValueError(f'Group {group} not registered.') binding = { @@ -124,6 +138,15 @@ def bind(self, keys, group, desc, callback): self.groups[group]['bindings'].append(binding) def add_group(self, group, desc): + """Add a new binding group. + + Args: + group: + The name of the group. + desc: + A description of the group, used to generate help. + + """ self.groups[group] = { 'name': group, 'desc': desc, @@ -134,6 +157,12 @@ def __iter__(self): return iter(self.bindings) def list_groups(self): + """List groups and the bindings in them. + + Yields: + (group_name, [binding, ...]) + + """ for name, group in self.groups.items(): yield ( group, @@ -143,6 +172,3 @@ def list_groups(self): if binding['group'] == name ] ) - - -BINDINGS = Bindings() diff --git a/cylc/flow/tui/app.py b/cylc/flow/tui/app.py index cf09f75db0e..63e7fb36803 100644 --- a/cylc/flow/tui/app.py +++ b/cylc/flow/tui/app.py @@ -16,32 +16,23 @@ # along with this program. If not, see . """The application control logic for Tui.""" -import sys +from copy import deepcopy +from contextlib import contextmanager +from multiprocessing import Process +import re import urwid -from urwid import html_fragment from urwid.wimp import SelectableIcon -from pathlib import Path -from cylc.flow.network.client_factory import get_client -from cylc.flow.exceptions import ( - ClientError, - ClientTimeout, - WorkflowStopped -) -from cylc.flow.pathutil import get_workflow_run_dir +from cylc.flow.id import Tokens from cylc.flow.task_state import ( - TASK_STATUSES_ORDERED, TASK_STATUS_SUBMITTED, TASK_STATUS_RUNNING, TASK_STATUS_FAILED, ) -from cylc.flow.tui.data import ( - QUERY -) import cylc.flow.tui.overlay as overlay from cylc.flow.tui import ( - BINDINGS, + Bindings, FORE, BACK, JOB_COLOURS, @@ -49,32 +40,38 @@ ) from cylc.flow.tui.tree import ( find_closest_focus, - translate_collapsing + translate_collapsing, + expand_tree, +) +from cylc.flow.tui.updater import ( + Updater, + get_default_filters, ) from cylc.flow.tui.util import ( - compute_tree, dummy_flow, - get_task_status_summary, - get_workflow_status_str, render_node ) -from cylc.flow.workflow_files import WorkflowFiles +from cylc.flow.workflow_status import ( + WorkflowStatus, +) -urwid.set_encoding('utf8') # required for unicode task icons +# default workflow / task filters +# (i.e. show everything) +DEFAULT_FILTERS = get_default_filters() + -TREE_EXPAND_DEPTH = [2] +urwid.set_encoding('utf8') # required for unicode task icons class TuiWidget(urwid.TreeWidget): """Display widget for tree nodes. Arguments: + app (TuiApp): + Reference to the application. node (TuiNode): The root tree node. - max_depth (int): - Determines which nodes are unfolded by default. - The maximum tree depth to unfold. """ @@ -82,16 +79,12 @@ class TuiWidget(urwid.TreeWidget): # will skip rows when the user navigates unexpandable_icon = SelectableIcon(' ', 0) - def __init__(self, node, max_depth=None): - if not max_depth: - max_depth = TREE_EXPAND_DEPTH[0] + def __init__(self, app, node): + self.app = app self._node = node self._innerwidget = None self.is_leaf = not node.get_child_keys() - if max_depth > 0: - self.expanded = node.get_depth() < max_depth - else: - self.expanded = True + self.expanded = False widget = self.get_indented_widget() urwid.WidgetWrap.__init__(self, widget) @@ -103,18 +96,6 @@ def selectable(self): """ return self.get_node().get_value()['type_'] != 'job_info' - def _is_leaf(self): - """Return True if this node has no children - - Note: the `is_leaf` attribute doesn't seem to give the right - answer. - - """ - return ( - not hasattr(self, 'git_first_child') - or not self.get_first_child() - ) - def get_display_text(self): """Compute the text to display for a given node. @@ -147,8 +128,8 @@ def keypress(self, size, key): return key def get_indented_widget(self): + """Override the Urwid method to handle leaf nodes differently.""" if self.is_leaf: - self._innerwidget = urwid.Columns( [ ('fixed', 1, self.unexpandable_icon), @@ -158,40 +139,80 @@ def get_indented_widget(self): ) return self.__super.get_indented_widget() + def update_expanded_icon(self, subscribe=True): + """Update the +/- icon. -class TuiNode(urwid.TreeNode): - """Data storage object for leaf nodes.""" - - def load_widget(self): - return TuiWidget(self) + This method overrides the built-in urwid update_expanded_icon method + in order to add logic for subscribing and unsubscribing to workflows. + Args: + subscribe: + If True, then we will [un]subscribe to workflows when workflow + nodes are expanded/collapsed. If False, then these events will + be ignored. Note we set subscribe=False when we are translating + the expand/collapse status when rebuilding the tree after an + update. -class TuiParentNode(urwid.ParentNode): - """Data storage object for interior/parent nodes.""" + """ + if subscribe: + node = self.get_node() + value = node.get_value() + data = value['data'] + type_ = value['type_'] + if type_ == 'workflow': + if self.expanded: + self.app.updater.subscribe(data['id']) + self.app.expand_on_load.add(data['id']) + else: + self.app.updater.unsubscribe(data['id']) + return urwid.TreeWidget.update_expanded_icon(self) + + +class TuiNode(urwid.ParentNode): + """Data storage object for Tui tree nodes.""" + + def __init__(self, app, *args, **kwargs): + self.app = app + urwid.ParentNode.__init__(self, *args, **kwargs) def load_widget(self): - return TuiWidget(self) + return TuiWidget(self.app, self) def load_child_keys(self): # Note: keys are really indices. - data = self.get_value() - return range(len(data['children'])) + return range(len(self.get_value()['children'])) def load_child_node(self, key): - """Return either an TuiNode or TuiParentNode""" - childdata = self.get_value()['children'][key] - if 'children' in childdata: - childclass = TuiParentNode - else: - childclass = TuiNode - return childclass( - childdata, + """Return a TuiNode instance for child "key".""" + return TuiNode( + self.app, + self.get_value()['children'][key], parent=self, key=key, depth=self.get_depth() + 1 ) +@contextmanager +def updater_subproc(filters): + """Runs the Updater in its own process. + + The updater provides the data for Tui to render. Running the updater + in its own process removes its CPU load from the Tui app allowing + it to remain responsive whilst updates are being gathered as well as + decoupling the application update logic from the data update logic. + """ + # start the updater + updater = Updater() + p = Process(target=updater.start, args=(filters,)) + try: + p.start() + yield updater + finally: + updater.terminate() + p.join() + + class TuiApp: """An application to display a single Cylc workflow. @@ -206,16 +227,25 @@ class TuiApp: """ - UPDATE_INTERVAL = 1 - CLIENT_TIMEOUT = 1 + # the UI update interval + # NOTE: this is different from the data update interval + UPDATE_INTERVAL = 0.1 + # colours to be used throughout the application palette = [ ('head', FORE, BACK), ('body', FORE, BACK), ('foot', 'white', 'dark blue'), ('key', 'light cyan', 'dark blue'), - ('title', FORE, BACK, 'bold'), + ('title', 'default, bold', BACK), + ('header', 'dark gray', BACK), + ('header_key', 'dark gray, bold', BACK), ('overlay', 'black', 'light gray'), + # cylc logo colours + ('R', 'light red, bold', BACK), + ('Y', 'yellow, bold', BACK), + ('G', 'light green, bold', BACK), + ('B', 'light blue, bold', BACK), ] + [ # job colours (f'job_{status}', colour, BACK) for status, colour in JOB_COLOURS.items() @@ -227,49 +257,117 @@ class TuiApp: for status, spec in WORKFLOW_COLOURS.items() ] - def __init__(self, id_, screen=None): - self.id_ = id_ - self.client = None + def __init__(self, screen=None): self.loop = None - self.screen = None + self.screen = screen self.stack = 0 self.tree_walker = None + # store a reference to the bindings on the app to avoid cicular import + self.bindings = BINDINGS + # create the template - topnode = TuiParentNode(dummy_flow({'id': 'Loading...'})) + topnode = TuiNode(self, dummy_flow({'id': 'Loading...'})) self.listbox = urwid.TreeListBox(urwid.TreeWalker(topnode)) header = urwid.Text('\n') - footer = urwid.AttrWrap( - # urwid.Text(self.FOOTER_TEXT), - urwid.Text(list_bindings()), - 'foot' - ) + footer = urwid.AttrWrap(urwid.Text(list_bindings()), 'foot') self.view = urwid.Frame( urwid.AttrWrap(self.listbox, 'body'), header=urwid.AttrWrap(header, 'head'), footer=footer ) - self.filter_states = { - state: True - for state in TASK_STATUSES_ORDERED - } - if isinstance(screen, html_fragment.HtmlGenerator): - # the HtmlGenerator only captures one frame - # so we need to pre-populate the GUI before - # starting the event loop - self.update() + self.filters = get_default_filters() - def main(self): - """Start the event loop.""" - self.loop = urwid.MainLoop( - self.view, - self.palette, - unhandled_input=self.unhandled_input, - screen=self.screen - ) - # schedule the first update - self.loop.set_alarm_in(0, self._update) - self.loop.run() + @contextmanager + def main(self, w_id=None, id_filter=None, interactive=True): + """Start the Tui app. + + With interactive=False, this does not start the urwid event loop to + make testing more deterministic. If you want Tui to update (i.e. + display the latest data), you must call the update method manually. + + Note, we still run the updater asynchronously in a subprocess so that + we can test the interactions between the Tui application and the + updater processes. + + """ + self.set_initial_filters(w_id, id_filter) + + with updater_subproc(self.filters) as updater: + self.updater = updater + + # pre-subscribe to the provided workflow if requested + self.expand_on_load = {w_id or 'root'} + if w_id: + self.updater.subscribe(w_id) + + # configure the urwid main loop + self.loop = urwid.MainLoop( + self.view, + self.palette, + unhandled_input=self.unhandled_input, + screen=self.screen + ) + + if interactive: + # Tui is being run normally as an interactive application + # schedule the first update + self.loop.set_alarm_in(0, self.update) + + # start the urwid main loop + try: + self.loop.run() + except KeyboardInterrupt: + yield + return + else: + # wait for the first full update + self.wait_until_loaded(w_id or 'root') + + yield self + + def set_initial_filters(self, w_id, id_filter): + """Set the default workflow/task filters on startup.""" + if w_id: + # Tui has been configured to look at a single workflow + # => filter everything else out + workflow = str(Tokens(w_id)['workflow']) + self.filters['workflows']['id'] = rf'^{re.escape(workflow)}$' + elif id_filter: + # a custom workflow ID filter has been provided + self.filters['workflows']['id'] = id_filter + + def wait_until_loaded(self, *ids, retries=None): + """Wait for any requested nodes to be created. + + Warning: + This method is blocking! It's for HTML / testing purposes only! + + Args: + ids: + Iterable containing the node IDs you want to wait for. + Note, these should be full IDs i.e. they should include the + user. + To wait for the root node to load, use "root". + retries: + The maximum number of updates to perform whilst waiting + for the specified IDs to appear in the tree. + + Returns: + A list of the IDs which NOT not appear in the store. + + """ + from time import sleep + ids = set(ids) + self.expand_on_load.update(ids) + try_ = 0 + while ids & self.expand_on_load: + self.update() + sleep(0.1) # blocking + try_ += 1 + if retries is not None and try_ > retries: + return list(self.expand_on_load) + return None def unhandled_input(self, key): """Catch key presses, uncaught events are passed down the chain.""" @@ -285,74 +383,6 @@ def unhandled_input(self, key): meth(self, *args) return - def get_snapshot(self): - """Contact the workflow, return a tree structure - - In the event of error contacting the workflow the - message is written to this Widget's header. - - Returns: - dict if successful, else False - - """ - try: - if not self.client: - self.client = get_client(self.id_, timeout=self.CLIENT_TIMEOUT) - data = self.client( - 'graphql', - { - 'request_string': QUERY, - 'variables': { - # list of task states we want to see - 'taskStates': [ - state - for state, is_on in self.filter_states.items() - if is_on - ] - } - } - ) - except WorkflowStopped: - # Distinguish stopped flow from non-existent flow. - self.client = None - full_path = Path(get_workflow_run_dir(self.id_)) - if ( - (full_path / WorkflowFiles.SUITE_RC).is_file() - or (full_path / WorkflowFiles.FLOW_FILE).is_file() - ): - message = "stopped" - else: - message = ( - f"No {WorkflowFiles.SUITE_RC} or {WorkflowFiles.FLOW_FILE}" - f"found in {self.id_}." - ) - - return dummy_flow({ - 'name': self.id_, - 'id': self.id_, - 'status': message, - 'stateTotals': {} - }) - except (ClientError, ClientTimeout) as exc: - # catch network / client errors - self.set_header([('workflow_error', str(exc))]) - return False - - if isinstance(data, list): - # catch GraphQL errors - try: - message = data[0]['error']['message'] - except (IndexError, KeyError): - message = str(data) - self.set_header([('workflow_error', message)]) - return False - - if len(data['workflows']) != 1: - # multiple workflows in returned data - shouldn't happen - raise ValueError() - - return compute_tree(data['workflows'][0]) - @staticmethod def get_node_id(node): """Return a unique identifier for a node. @@ -366,62 +396,82 @@ def get_node_id(node): """ return node.get_value()['id_'] - def set_header(self, message: list): - """Set the header message for this widget. + def update_header(self): + """Update the application header.""" + header = [ + # the Cylc Tui logo + ('R', 'C'), + ('Y', 'y'), + ('G', 'l'), + ('B', 'c'), + ('title', ' Tui') + ] + if self.filters['tasks'] != DEFAULT_FILTERS['tasks']: + # if task filters are active, display short help + header.extend([ + ('header', ' tasks filtered ('), + ('header_key', 'F'), + ('header', ' - edit, '), + ('header_key', 'R'), + ('header', ' - reset)'), + ]) + if self.filters['workflows'] != DEFAULT_FILTERS['workflows']: + # if workflow filters are active, display short help + header.extend([ + ('header', ' workflows filtered ('), + ('header_key', 'W'), + ('header', ' - edit, '), + ('header_key', 'E'), + ('header', ' - reset)'), + ]) + elif self.filters == DEFAULT_FILTERS: + # if not filters are available show application help + header.extend([ + ('header', ' '), + ('header_key', 'h'), + ('header', ' to show help, '), + ('header_key', 'q'), + ('header', ' to quit'), + ]) - Arguments: - message (object): - Text content for the urwid.Text widget, - may be a string, tuple or list, see urwid docs. - - """ # put in a one line gap - message.append('\n') - - # TODO: remove once Tui is delta-driven - # https://github.com/cylc/cylc-flow/issues/3527 - message.extend([ - ( - 'workflow_error', - 'TUI is experimental and may break with large flows' - ), - '\n' - ]) + header.append('\n') - self.view.header = urwid.Text(message) + # replace the previous header + self.view.header = urwid.Text(header) - def _update(self, *_): - try: - self.update() - except Exception as exc: - sys.exit(exc) + def get_update(self): + """Fetch the most recent update. - def update(self): + Returns the update, or False if there is no update queued. + """ + update = False + while not self.updater.update_queue.empty(): + # fetch the most recent update + update = self.updater.update_queue.get() + return update + + def update(self, *_): """Refresh the data and redraw this widget. Preserves the current focus and collapse/expand state. """ - # update the data store - # TODO: this can be done incrementally using deltas - # once this interface is available - snapshot = self.get_snapshot() - if snapshot is False: + # attempt to fetch an update + update = self.get_update() + if update is False: + # there was no update, try again later + if self.loop: + self.loop.set_alarm_in(self.UPDATE_INTERVAL, self.update) return False - # update the workflow status message - header = [get_workflow_status_str(snapshot['data'])] - status_summary = get_task_status_summary(snapshot['data']) - if status_summary: - header.extend([' ('] + status_summary + [' )']) - if not all(self.filter_states.values()): - header.extend([' ', '*filtered* "R" to reset', ' ']) - self.set_header(header) + # update the application header + self.update_header() # global update - the nuclear option - slow but simple # TODO: this can be done incrementally by adding and # removing nodes from the existing tree - topnode = TuiParentNode(snapshot) + topnode = TuiNode(self, update) # NOTE: because we are nuking the tree we need to manually # preserve the focus and collapse status of tree nodes @@ -443,9 +493,15 @@ def update(self): # preserve the collapse/expand status of all nodes translate_collapsing(self, old_node, new_node) + # expand any nodes which have been requested + for id_ in list(self.expand_on_load): + depth = 1 if id_ == 'root' else None + if expand_tree(self, new_node, id_, depth): + self.expand_on_load.remove(id_) + # schedule the next run of this update method if self.loop: - self.loop.set_alarm_in(self.UPDATE_INTERVAL, self._update) + self.loop.set_alarm_in(self.UPDATE_INTERVAL, self.update) return True @@ -457,14 +513,40 @@ def filter_by_task_state(self, filtered_state=None): A task state to filter by or None. """ - self.filter_states = { + self.filters['tasks'] = { state: (state == filtered_state) or not filtered_state - for state in self.filter_states + for state in self.filters['tasks'] } - return + self.updater.update_filters(self.filters) + + def reset_workflow_filters(self): + """Reset workflow state/id filters.""" + self.filters['workflows'] = deepcopy(DEFAULT_FILTERS['workflows']) + self.updater.update_filters(self.filters) + + def filter_by_workflow_state(self, *filtered_states): + """Filter workflows. + + Args: + filtered_state (str): + A task state to filter by or None. + + """ + for state in self.filters['workflows']: + if state != 'id': + self.filters['workflows'][state] = ( + (state in filtered_states) or not filtered_states + ) + self.updater.update_filters(self.filters) def open_overlay(self, fcn): - self.create_overlay(*fcn(self)) + """Open an overlay over the application. + + Args: + fcn: A function which returns an urwid widget to overlay. + + """ + self.create_overlay(*fcn(app=self)) def create_overlay(self, widget, kwargs): """Open an overlay over the monitor. @@ -521,6 +603,10 @@ def close_topmost(self): self.stack -= 1 +# register key bindings +# * all bindings must belong to a group +# * all keys are auto-documented in the help screen and application footer +BINDINGS = Bindings() BINDINGS.add_group( '', 'Application Controls' @@ -603,40 +689,68 @@ def close_topmost(self): ) BINDINGS.add_group( - 'filter', + 'filter tasks', 'Filter by task state' ) BINDINGS.bind( - ('F',), - 'filter', + ('T',), + 'filter tasks', 'Select task states to filter by', (TuiApp.open_overlay, overlay.filter_task_state) ) BINDINGS.bind( ('f',), - 'filter', + 'filter tasks', 'Show only failed tasks', (TuiApp.filter_by_task_state, TASK_STATUS_FAILED) ) BINDINGS.bind( ('s',), - 'filter', + 'filter tasks', 'Show only submitted tasks', (TuiApp.filter_by_task_state, TASK_STATUS_SUBMITTED) ) BINDINGS.bind( ('r',), - 'filter', + 'filter tasks', 'Show only running tasks', (TuiApp.filter_by_task_state, TASK_STATUS_RUNNING) ) BINDINGS.bind( ('R',), - 'filter', + 'filter tasks', 'Reset task state filtering', (TuiApp.filter_by_task_state,) ) +BINDINGS.add_group( + 'filter workflows', + 'Filter by workflow state' +) +BINDINGS.bind( + ('W',), + 'filter workflows', + 'Select workflow states to filter by', + (TuiApp.open_overlay, overlay.filter_workflow_state) +) +BINDINGS.bind( + ('E',), + 'filter workflows', + 'Reset workflow filtering', + (TuiApp.reset_workflow_filters,) +) +BINDINGS.bind( + ('p',), + 'filter workflows', + 'Show only running workflows', + ( + TuiApp.filter_by_workflow_state, + WorkflowStatus.RUNNING.value, + WorkflowStatus.PAUSED.value, + WorkflowStatus.STOPPING.value + ) +) + def list_bindings(): """Write out an in-line list of the key bindings.""" diff --git a/cylc/flow/tui/data.py b/cylc/flow/tui/data.py index 4fd538d8783..04c3d7220b1 100644 --- a/cylc/flow/tui/data.py +++ b/cylc/flow/tui/data.py @@ -16,19 +16,27 @@ from functools import partial from subprocess import Popen, PIPE -import sys -from cylc.flow.exceptions import ClientError +from cylc.flow.exceptions import ( + ClientError, + ClientTimeout, + WorkflowStopped, +) +from cylc.flow.network.client_factory import get_client +from cylc.flow.id import Tokens from cylc.flow.tui.util import ( extract_context ) +# the GraphQL query which Tui runs against each of the workflows +# is is subscribed to QUERY = ''' query cli($taskStates: [String]){ workflows { id name + port status stateTotals taskProxies(states: $taskStates) { @@ -82,6 +90,7 @@ } ''' +# the list of mutations we can call on a running scheduler MUTATIONS = { 'workflow': [ 'pause', @@ -107,35 +116,23 @@ ] } +# mapping of Tui's node types (e.g. workflow) onto GraphQL argument types +# (e.g. WorkflowID) ARGUMENT_TYPES = { + # : 'workflow': '[WorkflowID]!', 'task': '[NamespaceIDGlob]!', } -MUTATION_TEMPLATES = { - 'workflow': ''' - mutation($workflow: [WorkflowID]!) { - pause (workflows: $workflow) { - result - } - } - ''', - 'task': ''' - mutation($workflow: [WorkflowID]!, $task: [NamespaceIDGlob]!) { - trigger (workflows: $workflow, tasks: $task) { - result - } - } - ''' -} - -def cli_cmd(*cmd): +def cli_cmd(*cmd, ret=False): """Issue a CLI command. Args: cmd: The command without the 'cylc' prefix'. + ret: + If True, the stdout will be returned. Rasies: ClientError: @@ -149,28 +146,136 @@ def cli_cmd(*cmd): stdout=PIPE, text=True, ) - out, err = proc.communicate() + _out, err = proc.communicate() if proc.returncode != 0: - raise ClientError(f'Error in command {" ".join(cmd)}\n{err}') + raise ClientError(f'Error in command cylc {" ".join(cmd)}\n{err}') + if ret: + return _out + + +def _show(id_): + """Special mutation to display cylc show output.""" + # dynamic import to avoid circular import issues + from cylc.flow.tui.overlay import text_box + return partial( + text_box, + text=cli_cmd('show', id_, '--color=never', ret=True), + ) + + +def _log(id_): + """Special mutation to open the log view.""" + # dynamic import to avoid circular import issues + from cylc.flow.tui.overlay import log + return partial( + log, + id_=id_, + list_files=partial(_list_log_files, id_), + get_log=partial(_get_log, id_), + ) + + +def _parse_log_header(contents): + """Parse the cat-log header. + + The "--prepend-path" option to "cat-log" adds a line containing the host + and path to the file being viewed in the form: + + # : + + Args: + contents: + The raw log file contents as returned by "cat-log". + Returns: + tuple - (host, path, text) -def _clean(workflow): - # for now we will exit tui when the workflow is cleaned - # this will change when tui supports multiple workflows - cli_cmd('clean', workflow) - sys.exit(0) + host: + The host where the file was retrieved from. + path: + The absolute path to the log file. + text: + The log file contents with the header removed. + + """ + contents, text = contents.split('\n', 1) + contents = contents.replace('# ', '') + host, path = contents.split(':') + return host, path, text + + +def _get_log(id_, filename=None): + """Retrieve the contents of a log file. + + Args: + id_: + The Cylc universal ID of the thing you want to fetch the log file + for. + filename: + The file name to retrieve (note name not path). + If "None", then the default log file will be retrieved. + + """ + cmd = [ + 'cat-log', + '--mode=cat', + '--prepend-path', + ] + if filename: + cmd.append(f'--file={filename}') + text = cli_cmd( + *cmd, + id_, + ret=True, + ) + return _parse_log_header(text) + + +def _list_log_files(id_): + """Return a list of available log files. + + Args: + id_: + The Cylc universal ID of the thing you want to fetch the log file + for. + + """ + text = cli_cmd('cat-log', '--mode=list-dir', id_, ret=True) + return text.splitlines() +# the mutations we have to go through the CLI to perform OFFLINE_MUTATIONS = { + 'user': { + 'stop-all': partial(cli_cmd, 'stop', '*'), + }, 'workflow': { 'play': partial(cli_cmd, 'play'), - 'clean': _clean, + 'clean': partial(cli_cmd, 'clean', '--yes'), 'reinstall-reload': partial(cli_cmd, 'vr', '--yes'), - } + 'log': _log, + }, + 'task': { + 'log': _log, + 'show': _show, + }, + 'job': { + 'log': _log, + }, } def generate_mutation(mutation, arguments): + """Return a GraphQL mutation string. + + Args: + mutation: + The mutation name. + Arguments: + The arguments to provide to it. + + """ + arguments.pop('user') graphql_args = ', '.join([ f'${argument}: {ARGUMENT_TYPES[argument]}' for argument in arguments @@ -189,11 +294,25 @@ def generate_mutation(mutation, arguments): ''' -def list_mutations(client, selection): +def list_mutations(selection, is_running=True): + """List mutations relevant to the provided selection. + + Args: + selection: + The user selection. + is_running: + If False, then mutations which require the scheduler to be + running will be omitted. + + Note, this is only relevant for workflow nodes because if a + workflow is stopped, then any tasks within it will be removed + anyway. + + """ context = extract_context(selection) selection_type = list(context)[-1] ret = [] - if client: + if is_running: # add the online mutations ret.extend(MUTATIONS.get(selection_type, [])) # add the offline mutations @@ -201,47 +320,99 @@ def list_mutations(client, selection): return sorted(ret) -def context_to_variables(context): +def context_to_variables(context, jobs=False): """Derive multiple selection out of single selection. + Note, this interface exists with the aim of facilitating the addition of + multiple selection at a later date. + Examples: >>> context_to_variables(extract_context(['~a/b//c/d'])) - {'workflow': ['b'], 'task': ['c/d']} + {'user': ['a'], 'workflow': ['b'], 'task': ['c/d']} >>> context_to_variables(extract_context(['~a/b//c'])) - {'workflow': ['b'], 'task': ['c/*']} + {'user': ['a'], 'workflow': ['b'], 'task': ['c/*']} >>> context_to_variables(extract_context(['~a/b'])) - {'workflow': ['b']} + {'user': ['a'], 'workflow': ['b']} + + # Note, jobs are omitted by default + >>> context_to_variables(extract_context(['~a/b//c/d/01'])) + {'user': ['a'], 'workflow': ['b'], 'task': ['c/d']} + + # This is because Cylc commands cannot generally operate on jobs only + # tasks. + # To let jobs slide through: + >>> context_to_variables(extract_context(['~a/b//c/d/01']), jobs=True) + {'user': ['a'], 'workflow': ['b'], 'job': ['c/d/01']} """ # context_to_variables because it can only handle single-selection ATM - variables = {'workflow': context['workflow']} - if 'task' in context: + variables = {'user': context['user']} + + if 'workflow' in context: + variables['workflow'] = context['workflow'] + if jobs and 'job' in context: + variables['job'] = [ + Tokens( + cycle=context['cycle'][0], + task=context['task'][0], + job=context['job'][0], + ).relative_id + ] + elif 'task' in context: variables['task'] = [ - f'{context["cycle"][0]}/{context["task"][0]}' + Tokens( + cycle=context['cycle'][0], + task=context['task'][0] + ).relative_id ] elif 'cycle' in context: - variables['task'] = [f'{context["cycle"][0]}/*'] + variables['task'] = [ + Tokens(cycle=context['cycle'][0], task='*').relative_id + ] return variables -def mutate(client, mutation, selection): - if mutation in OFFLINE_MUTATIONS['workflow']: - offline_mutate(mutation, selection) - elif client: - online_mutate(client, mutation, selection) +def mutate(mutation, selection): + """Call a mutation. + + Args: + mutation: + The mutation name (e.g. stop). + selection: + The Tui selection (i.e. the row(s) selected in Tui). + + """ + if mutation in { + _mutation + for section in OFFLINE_MUTATIONS.values() + for _mutation in section + }: + return offline_mutate(mutation, selection) else: - raise Exception( - f'Cannot peform command {mutation} on a stopped workflow' - ' or invalid command.' - ) + online_mutate(mutation, selection) + return None -def online_mutate(client, mutation, selection): +def online_mutate(mutation, selection): """Issue a mutation over a network interface.""" context = extract_context(selection) variables = context_to_variables(context) + + # note this only supports single workflow mutations at present + workflow = variables['workflow'][0] + try: + client = get_client(workflow) + except WorkflowStopped: + raise Exception( + f'Cannot peform command {mutation} on a stopped workflow' + ) + except (ClientError, ClientTimeout) as exc: + raise Exception( + f'Error connecting to workflow: {exc}' + ) + request_string = generate_mutation(mutation, variables) client( 'graphql', @@ -255,7 +426,21 @@ def online_mutate(client, mutation, selection): def offline_mutate(mutation, selection): """Issue a mutation over the CLI or other offline interface.""" context = extract_context(selection) - variables = context_to_variables(context) - for workflow in variables['workflow']: - # NOTE: this currently only supports workflow mutations - OFFLINE_MUTATIONS['workflow'][mutation](workflow) + variables = context_to_variables(context, jobs=True) + if 'job' in variables: + for job in variables['job']: + id_ = Tokens(job, relative=True).duplicate( + workflow=variables['workflow'][0] + ) + return OFFLINE_MUTATIONS['job'][mutation](id_.id) + if 'task' in variables: + for task in variables['task']: + id_ = Tokens(task, relative=True).duplicate( + workflow=variables['workflow'][0] + ) + return OFFLINE_MUTATIONS['task'][mutation](id_.id) + if 'workflow' in variables: + for workflow in variables['workflow']: + return OFFLINE_MUTATIONS['workflow'][mutation](workflow) + else: + return OFFLINE_MUTATIONS['user'][mutation]() diff --git a/cylc/flow/tui/overlay.py b/cylc/flow/tui/overlay.py index cddcc8d5093..94c1a8b2ef5 100644 --- a/cylc/flow/tui/overlay.py +++ b/cylc/flow/tui/overlay.py @@ -38,19 +38,17 @@ """ from functools import partial +import re import sys import urwid -from cylc.flow.exceptions import ( - ClientError, -) +from cylc.flow.id import Tokens from cylc.flow.task_state import ( TASK_STATUSES_ORDERED, TASK_STATUS_WAITING ) from cylc.flow.tui import ( - BINDINGS, JOB_COLOURS, JOB_ICON, TUI @@ -60,32 +58,109 @@ mutate, ) from cylc.flow.tui.util import ( - get_task_icon + get_task_icon, + get_text_dimensions, ) +def _get_display_id(id_): + """Return an ID for display in context menus. + + * Display the full ID for users/workflows + * Display the relative ID for everything else + + """ + tokens = Tokens(id_) + if tokens.is_task_like: + # if it's a cycle/task/job, then use the relative id + return tokens.relative_id + else: + # otherwise use the full id + return tokens.id + + +def _toggle_filter(app, filter_group, status, *_): + """Toggle a filter state.""" + app.filters[filter_group][status] = not app.filters[filter_group][status] + app.updater.update_filters(app.filters) + + +def _invert_filter(checkboxes, *_): + """Invert the state of all filters.""" + for checkbox in checkboxes: + checkbox.set_state(not checkbox.state) + + +def filter_workflow_state(app): + """Return a widget for adjusting the workflow filter options.""" + checkboxes = [ + urwid.CheckBox( + [status], + state=is_on, + on_state_change=partial(_toggle_filter, app, 'workflows', status) + ) + for status, is_on in app.filters['workflows'].items() + if status != 'id' + ] + + workflow_id_prompt = 'id (regex)' + + def update_id_filter(widget, value): + nonlocal app + try: + # ensure the filter is value before updating the filter + re.compile(value) + except re.error: + # error in the regex -> inform the user + widget.set_caption(f'{workflow_id_prompt} - error: \n') + else: + # valid regex -> update the filter + widget.set_caption(f'{workflow_id_prompt}: \n') + app.filters['workflows']['id'] = value + app.updater.update_filters(app.filters) + + id_filter_widget = urwid.Edit( + caption=f'{workflow_id_prompt}: \n', + edit_text=app.filters['workflows']['id'], + ) + urwid.connect_signal(id_filter_widget, 'change', update_id_filter) + + widget = urwid.ListBox( + urwid.SimpleFocusListWalker([ + urwid.Text('Filter Workflow States'), + urwid.Divider(), + urwid.Padding( + urwid.Button( + 'Invert', + on_press=partial(_invert_filter, checkboxes) + ), + right=19 + ) + ] + checkboxes + [ + urwid.Divider(), + id_filter_widget, + ]) + ) + + return ( + widget, + {'width': 35, 'height': 23} + ) + + def filter_task_state(app): """Return a widget for adjusting the task state filter.""" - def toggle(state, *_): - """Toggle a filter state.""" - app.filter_states[state] = not app.filter_states[state] - checkboxes = [ urwid.CheckBox( get_task_icon(state) + [' ' + state], state=is_on, - on_state_change=partial(toggle, state) + on_state_change=partial(_toggle_filter, app, 'tasks', state) ) - for state, is_on in app.filter_states.items() + for state, is_on in app.filters['tasks'].items() ] - def invert(*_): - """Invert the state of all filters.""" - for checkbox in checkboxes: - checkbox.set_state(not checkbox.state) - widget = urwid.ListBox( urwid.SimpleFocusListWalker([ urwid.Text('Filter Task States'), @@ -93,7 +168,7 @@ def invert(*_): urwid.Padding( urwid.Button( 'Invert', - on_press=invert + on_press=partial(_invert_filter, checkboxes) ), right=19 ) @@ -127,7 +202,7 @@ def help_info(app): ] # list key bindings - for group, bindings in BINDINGS.list_groups(): + for group, bindings in app.bindings.list_groups(): items.append( urwid.Text([ f'{group["desc"]}:' @@ -215,21 +290,37 @@ def context(app): value = app.tree_walker.get_focus()[0].get_node().get_value() selection = [value['id_']] # single selection ATM + is_running = True + if ( + value['type_'] == 'workflow' + and value['data']['status'] not in {'running', 'paused'} + ): + # this is a stopped workflow + # => don't display mutations only valid for a running workflow + is_running = False + def _mutate(mutation, _): - nonlocal app + nonlocal app, selection + app.open_overlay(partial(progress, text='Running Command')) + overlay_fcn = None try: - mutate(app.client, mutation, selection) - except ClientError as exc: + overlay_fcn = mutate(mutation, selection) + except Exception as exc: app.open_overlay(partial(error, text=str(exc))) else: app.close_topmost() app.close_topmost() + if overlay_fcn: + app.open_overlay(overlay_fcn) + + # determine the ID to display for the context menu + display_id = _get_display_id(value['id_']) widget = urwid.ListBox( urwid.SimpleFocusListWalker( [ - urwid.Text(f'id: {value["id_"]}'), + urwid.Text(f'id: {display_id}'), urwid.Divider(), urwid.Text('Action'), urwid.Button( @@ -242,14 +333,17 @@ def _mutate(mutation, _): mutation, on_press=partial(_mutate, mutation) ) - for mutation in list_mutations(app.client, selection) + for mutation in list_mutations( + selection, + is_running, + ) ] ) ) return ( widget, - {'width': 30, 'height': 20} + {'width': 50, 'height': 20} ) @@ -273,3 +367,105 @@ def progress(app, text='Working'): ]), {'width': 30, 'height': 10} ) + + +def log(app, id_=None, list_files=None, get_log=None): + """An overlay for displaying log files.""" + # display the host name where the file is coming from + host_widget = urwid.Text('loading...') + # display the log filepath + file_widget = urwid.Text('') + # display the actual log file itself + text_widget = urwid.Text('') + + def open_menu(*_args, **_kwargs): + """Open an overlay for selecting a log file.""" + nonlocal app, id_ + app.open_overlay(select_log) + + def select_log(*_args, **_kwargs): + """Create an overlay for selecting a log file.""" + nonlocal list_files, id_ + try: + files = list_files() + except Exception as exc: + return error(app, text=str(exc)) + return ( + urwid.ListBox([ + *[ + urwid.Text('Select File'), + urwid.Divider(), + ], + *[ + urwid.Button( + filename, + on_press=partial( + open_log, + filename=filename, + close=True, + ), + ) + for filename in files + ], + ]), + # NOTE: the "+6" avoids the need for scrolling + {'width': 40, 'height': len(files) + 6} + ) + + def open_log(*_, filename=None, close=False): + """View the provided log file. + + Args: + filename: + The name of the file to open (note name not path). + close: + If True, then the topmost overlay will be closed when a file is + selected. Use this to close the "select_log" overlay. + + """ + + nonlocal host_widget, file_widget, text_widget + try: + host, path, text = get_log(filename) + except Exception as exc: + host_widget.set_text(f'Error: {exc}') + file_widget.set_text('') + text_widget.set_text('') + else: + host_widget.set_text(f'Host: {host}') + file_widget.set_text(f'Path: {path}') + text_widget.set_text(text) + if close: + app.close_topmost() + + # load the default log file + if id_: + # NOTE: the kwargs are not provided in the overlay unit tests + open_log() + + return ( + urwid.ListBox([ + host_widget, + file_widget, + urwid.Button( + 'Select File', + on_press=open_menu, + ), + urwid.Divider(), + text_widget, + ]), + # open full screen + {'width': 9999, 'height': 9999} + ) + + +def text_box(app, text=''): + """A simple text box overlay.""" + width, height = get_text_dimensions(text) + return ( + urwid.ListBox([ + urwid.Text(text), + ]), + # NOTE: those fudge factors account for the overlay border & padding + {'width': width + 4, 'height': height + 6} + ) diff --git a/cylc/flow/tui/tree.py b/cylc/flow/tui/tree.py index 84ed55ab53a..02cecee61ed 100644 --- a/cylc/flow/tui/tree.py +++ b/cylc/flow/tui/tree.py @@ -56,6 +56,78 @@ def find_closest_focus(app, old_node, new_node): ) +def expand_tree(app, tree_node, id_, depth=5, node_types=None): + """Expand the Tui tree to the desired level. + + Arguments: + app: + The Tui application instance. + tree_node: + The Tui widget representing the tree view. + id_: + If specified, we will look within the tree for a node matching + this ID and the tree below this node will be expanded. + depth: + The max depth to expand nodes too. + node_types: + Whitelist of node types to expand, note "task", "job" and "spring" + nodes are excluded by default. + + Returns: + True, if the node was found in the tree, is loaded and has been + expanded. + + Examples: + # expand the top three levels of the tree + compute_tree(app, node, None, 3) + + # expand the "root" node AND the top five levels of the tree under + # ~user/workflow + compute_tree(app, node, '~user/workflow') + + """ + if not node_types: + # don't auto-expand job nodes by default + node_types = {'root', 'workflow', 'cycle', 'family'} + + root_node = tree_node.get_root() + requested_node = root_node + + # locate the "id_" within the tree if specified + if id_: + for node in walk_tree(root_node): + key = app.get_node_id(node) + if key == id_: + requested_node = node + child_keys = node.get_child_keys() + if ( + # if the node only has one child + len(child_keys) == 1 + # and that child is a "#spring" node (i.e. a loading node) + and ( + node.get_child_node(0).get_value()['type_'] + ) == '#spring' + ): + # then the content hasn't loaded yet so the node cannot be + # expanded + return False + break + else: + # the requested node does not exist yet + # it might still be loading + return False + + # expand the specified nodes + for node in (*walk_tree(requested_node, depth), root_node): + if node.get_value()['type_'] not in node_types: + continue + widget = node.get_widget() + widget.expanded = True + widget.update_expanded_icon(False) + + return True + + def translate_collapsing(app, old_node, new_node): """Transfer the collapse state from one tree to another. @@ -81,29 +153,46 @@ def translate_collapsing(app, old_node, new_node): for node in walk_tree(new_root): key = app.get_node_id(node) if key in old_tree: + # this node was present before + # => translate its expansion to the new tree expanded = old_tree.get(key) widget = node.get_widget() if widget.expanded != expanded: widget.expanded = expanded - widget.update_expanded_icon() - - -def walk_tree(node): + widget.update_expanded_icon(False) + else: + # this node was not present before + # => apply the standard expansion logic + expand_tree( + app, + node, + key, + 3, + # don't auto-expand workflows, only cycles/families + # and the root node to help expand the tree on startup + node_types={'root', 'cycle', 'family'} + ) + + +def walk_tree(node, depth=None): """Yield nodes in order. Arguments: node (urwid.TreeNode): Yield this node and all nodes beneath it. + depth: + The maximum depth to walk to or None to walk all children. Yields: urwid.TreeNode """ - stack = [node] + stack = [(node, 1)] while stack: - node = stack.pop() + node, _depth = stack.pop() yield node - stack.extend([ - node.get_child_node(index) - for index in node.get_child_keys() - ]) + if depth is None or _depth < depth: + stack.extend([ + (node.get_child_node(index), _depth + 1) + for index in node.get_child_keys() + ]) diff --git a/cylc/flow/tui/updater.py b/cylc/flow/tui/updater.py new file mode 100644 index 00000000000..5321a4d6c1f --- /dev/null +++ b/cylc/flow/tui/updater.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Contains the logic for updating the Tui app.""" + +from asyncio import ( + run, + sleep, + gather, +) +from contextlib import suppress +from copy import deepcopy +from getpass import getuser +from multiprocessing import Queue +from time import time + +from zmq.error import ZMQError + +from cylc.flow.exceptions import ( + ClientError, + ClientTimeout, + CylcError, + WorkflowStopped, +) +from cylc.flow.id import Tokens +from cylc.flow.network.client_factory import get_client +from cylc.flow.network.scan import ( + filter_name, + graphql_query, + is_active, + scan, +) +from cylc.flow.task_state import ( + TASK_STATUSES_ORDERED, +) +from cylc.flow.tui.data import ( + QUERY +) +from cylc.flow.tui.util import ( + compute_tree, + suppress_logging, +) +from cylc.flow.workflow_status import ( + WorkflowStatus, +) + + +ME = getuser() + + +def get_default_filters(): + """Return default task/workflow filters. + + These filters show everything. + """ + return { + 'tasks': { + # filtered task statuses + state: True + for state in TASK_STATUSES_ORDERED + }, + 'workflows': { + # filtered workflow statuses + **{ + state.value: True + for state in WorkflowStatus + }, + # filtered workflow ids + 'id': '.*', + } + } + + +class Updater(): + """The bit of Tui which provides the data. + + It lists workflows using the "scan" interface, and provides detail using + the "GraphQL" interface. + + """ + + # the maximum time to wait for a workflow update + CLIENT_TIMEOUT = 2 + + # the interval between workflow listing scans + BASE_SCAN_INTERVAL = 20 + + # the interval between workflow data updates + BASE_UPDATE_INTERVAL = 1 + + # the command signal used to tell the updater to shut down + SIGNAL_TERMINATE = 'terminate' + + def __init__(self): + # Cylc comms clients for each workflow we're connected to + self._clients = {} + + # iterate over this to get a list of workflows + self._scan_pipe = None + # the new pipe if the workflow filter options are changed + self.__scan_pipe = None + + # task/workflow filters + self.filters = None # note set on self.run() + # queue for pushing out updates + self.update_queue = Queue( + # block the updater if it gets too far ahead of the application + maxsize=10 + ) + # queue for commands to the updater + self._command_queue = Queue() + + def subscribe(self, w_id): + """Subscribe to updates from a workflow.""" + self._command_queue.put((self._subscribe.__name__, w_id)) + + def unsubscribe(self, w_id): + """Unsubscribe to updates from a workflow.""" + self._command_queue.put((self._unsubscribe.__name__, w_id)) + + def update_filters(self, filters): + """Update the task state filter.""" + self._command_queue.put((self._update_filters.__name__, filters)) + + def terminate(self): + """Stop the updater.""" + self._command_queue.put((self.SIGNAL_TERMINATE, None)) + + def start(self, filters): + """Start the updater in a new asyncio.loop. + + The Tui app will call this within a dedicated process. + """ + with suppress(KeyboardInterrupt): + run(self.run(filters)) + + async def run(self, filters): + """Start the updater in an existing asyncio.loop. + + The tests call this within the same process. + """ + with suppress_logging(): + self._update_filters(filters) + await self._update() + + def _subscribe(self, w_id): + if w_id not in self._clients: + self._clients[w_id] = None + + def _unsubscribe(self, w_id): + if w_id in self._clients: + self._clients.pop(w_id) + + def _update_filters(self, filters): + if ( + not self.filters + or filters['workflows']['id'] != self.filters['workflows']['id'] + ): + # update the scan pipe + self.__scan_pipe = ( + # scan all workflows + scan + | filter_name(filters['workflows']['id']) + # if the workflow is active, retrieve its status + | is_active(True, filter_stop=False) + | graphql_query({'status': None}) + ) + + self.filters = filters + + async def _update(self): + last_scan_time = 0 + while True: + # process any pending commands + if not self._command_queue.empty(): + (command, payload) = self._command_queue.get() + if command == self.SIGNAL_TERMINATE: + break + getattr(self, command)(payload) + continue + + # do a workflow scan if it's due + update_start_time = time() + if update_start_time - last_scan_time > self.BASE_SCAN_INTERVAL: + data = await self._scan() + + # get the next snapshot from workflows we are subscribed to + self.update_queue.put(await self._run_update(data)) + + # schedule the next update + update_time = time() - update_start_time + await sleep(self.BASE_UPDATE_INTERVAL - update_time) + + async def _run_update(self, data): + # copy the scanned data so it can be reused for future updates + data = deepcopy(data) + + # connect to schedulers if needed + self._connect(data) + + # update data with the response from each workflow + # NOTE: Currently we're bunching these updates together so Tui will + # only update as fast as the slowest responding workflow. + # We could run these updates separately if this is an issue. + await gather( + *( + self._update_workflow(w_id, client, data) + for w_id, client in self._clients.items() + ) + ) + + return compute_tree(data) + + async def _update_workflow(self, w_id, client, data): + if not client: + # we could not connect to this workflow + # e.g. workflow is shut down + return + + try: + # fetch the data from the workflow + workflow_update = await client.async_request( + 'graphql', + { + 'request_string': QUERY, + 'variables': { + # list of task states we want to see + 'taskStates': [ + state + for state, is_on in self.filters['tasks'].items() + if is_on + ] + } + } + ) + except WorkflowStopped: + # remove the client on any error, we'll reconnect next time + self._clients[w_id] = None + for workflow in data['workflows']: + if workflow['id'] == w_id: + break + else: + # there's no entry here, create a stub + # NOTE: this handles the situation where we've connected to a + # workflow before it has appeared in a scan which matters to + # the tests as they use fine timings + data['workflows'].append({ + 'id': w_id, + 'status': 'stopped', + }) + except (CylcError, ZMQError): + # something went wrong :( + # remove the client on any error, we'll reconnect next time + self._clients[w_id] = None + else: + # the data arrived, add it to the update + workflow_data = workflow_update['workflows'][0] + for workflow in data['workflows']: + if workflow['id'] == workflow_data['id']: + workflow.update(workflow_data) + break + + def _connect(self, data): + """Connect to all subscribed workflows.""" + for w_id, client in self._clients.items(): + if not client: + try: + self._clients[w_id] = get_client( + Tokens(w_id)['workflow'], + timeout=self.CLIENT_TIMEOUT + ) + except WorkflowStopped: + for workflow in data['workflows']: + if workflow['id'] == w_id: + workflow['_tui_data'] = 'Workflow is not running' + except (ZMQError, ClientError, ClientTimeout) as exc: + for workflow in data['workflows']: + if workflow['id'] == w_id: + workflow['_tui_data'] = f'Error: {exc}' + break + + async def _scan(self): + """Scan for workflows on the filesystem.""" + data = {'workflows': []} + workflow_filter_statuses = { + status + for status, filtered in self.filters['workflows'].items() + if filtered + } + if self.__scan_pipe: + # switch to the new pipe if it has been changed + self._scan_pipe = self.__scan_pipe + async for workflow in self._scan_pipe: + status = workflow.get('status', WorkflowStatus.STOPPED.value) + if status not in workflow_filter_statuses: + # this workflow is filtered out + continue + data['workflows'].append({ + 'id': f'~{ME}/{workflow["name"]}', + 'name': workflow['name'], + 'status': status, + 'stateTotals': {}, + }) + + data['workflows'].sort(key=lambda x: x['id']) + return data diff --git a/cylc/flow/tui/util.py b/cylc/flow/tui/util.py index 575fc693437..88e960e249a 100644 --- a/cylc/flow/tui/util.py +++ b/cylc/flow/tui/util.py @@ -16,10 +16,14 @@ # along with this program. If not, see . """Common utilities for Tui.""" +from contextlib import contextmanager +from getpass import getuser from itertools import zip_longest import re from time import time +from typing import Tuple +from cylc.flow import LOG from cylc.flow.id import Tokens from cylc.flow.task_state import ( TASK_STATUS_RUNNING @@ -33,6 +37,26 @@ from cylc.flow.wallclock import get_unix_time_from_time_string +# the Tui user, note this is always the same as the workflow owner +# (Tui doesn't do multi-user stuff) +ME = getuser() + + +@contextmanager +def suppress_logging(): + """Suppress Cylc logging. + + Log goes to stdout/err which can pollute Urwid apps. + Patching sys.stdout/err is insufficient so we set the level to something + silly for the duration of this context manager then set it back again + afterwards. + """ + level = LOG.getEffectiveLevel() + LOG.setLevel(99999) + yield + LOG.setLevel(level) + + def get_task_icon( status, *, @@ -113,80 +137,90 @@ def idpop(id_): return tokens.id -def compute_tree(flow): - """Digest GraphQL data to produce a tree. +def compute_tree(data): + """Digest GraphQL data to produce a tree.""" + root_node = add_node('root', 'root', {}, data={}) - Arguments: - flow (dict): - A dictionary representing a single workflow. + for flow in data['workflows']: + nodes = {} + flow_node = add_node( + 'workflow', flow['id'], nodes, data=flow) + root_node['children'].append(flow_node) - Returns: - dict - A top-level workflow node. + # populate cycle nodes + for cycle in flow.get('cyclePoints', []): + cycle['id'] = idpop(cycle['id']) # strip the family off of the id + cycle_node = add_node('cycle', cycle['id'], nodes, data=cycle) + flow_node['children'].append(cycle_node) - """ - nodes = {} - flow_node = add_node( - 'workflow', flow['id'], nodes, data=flow) - - # populate cycle nodes - for cycle in flow['cyclePoints']: - cycle['id'] = idpop(cycle['id']) # strip the family off of the id - cycle_node = add_node('cycle', cycle['id'], nodes, data=cycle) - flow_node['children'].append(cycle_node) - - # populate family nodes - for family in flow['familyProxies']: - add_node('family', family['id'], nodes, data=family) - - # create cycle/family tree - for family in flow['familyProxies']: - family_node = add_node( - 'family', family['id'], nodes) - first_parent = family['firstParent'] - if ( - first_parent - and first_parent['name'] != 'root' - ): - parent_node = add_node( - 'family', first_parent['id'], nodes) - parent_node['children'].append(family_node) - else: - add_node( - 'cycle', idpop(family['id']), nodes - )['children'].append(family_node) - - # add leaves - for task in flow['taskProxies']: - # If there's no first parent, the child will have been deleted - # during/after API query resolution. So ignore. - if not task['firstParent']: - continue - task_node = add_node( - 'task', task['id'], nodes, data=task) - if task['firstParent']['name'] == 'root': - family_node = add_node( - 'cycle', idpop(task['id']), nodes) - else: + # populate family nodes + for family in flow.get('familyProxies', []): + add_node('family', family['id'], nodes, data=family) + + # create cycle/family tree + for family in flow.get('familyProxies', []): family_node = add_node( - 'family', task['firstParent']['id'], nodes) - family_node['children'].append(task_node) - for job in task['jobs']: - job_node = add_node( - 'job', job['id'], nodes, data=job) - job_info_node = add_node( - 'job_info', job['id'] + '_info', nodes, data=job) - job_node['children'] = [job_info_node] - task_node['children'].append(job_node) - - # sort - for (type_, _), node in nodes.items(): - if type_ != 'task': - # NOTE: jobs are sorted by submit-num in the GraphQL query - node['children'].sort( - key=lambda x: NaturalSort(x['id_']) + 'family', family['id'], nodes) + first_parent = family['firstParent'] + if ( + first_parent + and first_parent['name'] != 'root' + ): + parent_node = add_node( + 'family', first_parent['id'], nodes) + parent_node['children'].append(family_node) + else: + add_node( + 'cycle', idpop(family['id']), nodes + )['children'].append(family_node) + + # add leaves + for task in flow.get('taskProxies', []): + # If there's no first parent, the child will have been deleted + # during/after API query resolution. So ignore. + if not task['firstParent']: + continue + task_node = add_node( + 'task', task['id'], nodes, data=task) + if task['firstParent']['name'] == 'root': + family_node = add_node( + 'cycle', idpop(task['id']), nodes) + else: + family_node = add_node( + 'family', task['firstParent']['id'], nodes) + family_node['children'].append(task_node) + for job in task['jobs']: + job_node = add_node( + 'job', job['id'], nodes, data=job) + job_info_node = add_node( + 'job_info', job['id'] + '_info', nodes, data=job) + job_node['children'] = [job_info_node] + task_node['children'].append(job_node) + + # sort + for (type_, _), node in nodes.items(): + if type_ != 'task': + # NOTE: jobs are sorted by submit-num in the GraphQL query + node['children'].sort( + key=lambda x: NaturalSort(x['id_']) + ) + + # spring nodes + if 'port' not in flow: + # the "port" field is only available via GraphQL + # so we are not connected to this workflow yet + flow_node['children'].append( + add_node( + '#spring', + '#spring', + nodes, + data={ + 'id': flow.get('_tui_data', 'Loading ...'), + } + ) ) - return flow_node + return root_node class NaturalSort: @@ -340,7 +374,7 @@ def get_task_status_summary(flow): state_totals = flow['stateTotals'] return [ [ - ('', ' '), + ' ', (f'job_{state}', str(state_totals[state])), (f'job_{state}', JOB_ICON) ] @@ -361,103 +395,126 @@ def get_workflow_status_str(flow): list - Text list for the urwid.Text widget. """ - status = flow['status'] + + +def _render_user(node, data): + return f'~{ME}' + + +def _render_job_info(node, data): + key_len = max(len(key) for key in data) + ret = [ + f'{key} {" " * (key_len - len(key))} {value}\n' + for key, value in data.items() + ] + ret[-1] = ret[-1][:-1] # strip trailing newline + return ret + + +def _render_job(node, data): return [ - ( - 'title', - flow['name'], - ), - ' - ', - ( - f'workflow_{status}', - status - ) + f'#{data["submitNum"]:02d} ', + get_job_icon(data['state']) ] -def render_node(node, data, type_): - """Render a tree node as text. +def _render_task(node, data): + start_time = None + mean_time = None + try: + # due to sorting this is the most recent job + first_child = node.get_child_node(0) + except IndexError: + first_child = None + + # progress information + if data['state'] == TASK_STATUS_RUNNING and first_child: + start_time = first_child.get_value()['data']['startedTime'] + mean_time = data['task']['meanElapsedTime'] + + # the task icon + ret = get_task_icon( + data['state'], + is_held=data['isHeld'], + is_queued=data['isQueued'], + is_runahead=data['isRunahead'], + start_time=start_time, + mean_time=mean_time + ) - Args: - node (MonitorNode): - The node to render. - data (dict): - Data associated with that node. - type_ (str): - The node type (e.g. `task`, `job`, `family`). + # the most recent job status + ret.append(' ') + if first_child: + state = first_child.get_value()['data']['state'] + ret += [(f'job_{state}', f'{JOB_ICON}'), ' '] - """ - if type_ == 'job_info': - key_len = max(len(key) for key in data) - ret = [ - f'{key} {" " * (key_len - len(key))} {value}\n' - for key, value in data.items() - ] - ret[-1] = ret[-1][:-1] # strip trailing newline - return ret + # the task name + ret.append(f'{data["name"]}') + return ret - if type_ == 'job': - return [ - f'#{data["submitNum"]:02d} ', - get_job_icon(data['state']) - ] - if type_ == 'task': - start_time = None - mean_time = None - try: - # due to sorting this is the most recent job - first_child = node.get_child_node(0) - except IndexError: - first_child = None - - # progress information - if data['state'] == TASK_STATUS_RUNNING and first_child: - start_time = first_child.get_value()['data']['startedTime'] - mean_time = data['task']['meanElapsedTime'] - - # the task icon - ret = get_task_icon( +def _render_family(node, data): + return [ + get_task_icon( data['state'], is_held=data['isHeld'], is_queued=data['isQueued'], - is_runahead=data['isRunahead'], - start_time=start_time, - mean_time=mean_time - ) + is_runahead=data['isRunahead'] + ), + ' ', + Tokens(data['id']).pop_token()[1] + ] - # the most recent job status - ret.append(' ') - if first_child: - state = first_child.get_value()['data']['state'] - ret += [(f'job_{state}', f'{JOB_ICON}'), ' '] - - # the task name - ret.append(f'{data["name"]}') - return ret - - if type_ in ['family', 'cycle']: - return [ - get_task_icon( - data['state'], - is_held=data['isHeld'], - is_queued=data['isQueued'], - is_runahead=data['isRunahead'] + +def _render_unknown(node, data): + try: + state_totals = get_task_status_summary(data) + status = data['status'] + status_msg = [ + ( + 'title', + _display_workflow_id(data), ), - ' ', - Tokens(data['id']).pop_token()[1] + ' - ', + ( + f'workflow_{status}', + status + ) ] + except KeyError: + return Tokens(data['id']).pop_token()[1] + + return [*status_msg, *state_totals] + - return Tokens(data['id']).pop_token()[1] +def _display_workflow_id(data): + return data['name'] -PARTS = [ - 'user', - 'workflow', - 'cycle', - 'task', - 'job' -] +RENDER_FUNCTIONS = { + 'user': _render_user, + 'root': _render_user, + 'job_info': _render_job_info, + 'job': _render_job, + 'task': _render_task, + 'cycle': _render_family, + 'family': _render_family, +} + + +def render_node(node, data, type_): + """Render a tree node as text. + + Args: + node (MonitorNode): + The node to render. + data (dict): + Data associated with that node. + type_ (str): + The node type (e.g. `task`, `job`, `family`). + + """ + return RENDER_FUNCTIONS.get(type_, _render_unknown)(node, data) def extract_context(selection): @@ -476,9 +533,18 @@ def extract_context(selection): {'user': ['a'], 'workflow': ['b'], 'cycle': ['c'], 'task': ['d'], 'job': ['e']} + >>> list(extract_context(['root']).keys()) + ['user'] + """ ret = {} for item in selection: + if item == 'root': + # special handling for the Tui "root" node + # (this represents the workflow owner which is always the same as + # user for Tui) + ret['user'] = ME + continue tokens = Tokens(item) for key, value in tokens.items(): if ( @@ -489,3 +555,24 @@ def extract_context(selection): if value not in lst: lst.append(value) return ret + + +def get_text_dimensions(text: str) -> Tuple[int, int]: + """Return the monospace size of a block of multiline text. + + Examples: + >>> get_text_dimensions('foo') + (3, 1) + + >>> get_text_dimensions(''' + ... foo bar + ... baz + ... ''') + (11, 3) + + >>> get_text_dimensions('') + (0, 0) + + """ + lines = text.splitlines() + return max((0, *(len(line) for line in lines))), len(lines) diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py index 091d1712429..e9402ba058b 100644 --- a/cylc/flow/workflow_db_mgr.py +++ b/cylc/flow/workflow_db_mgr.py @@ -25,7 +25,6 @@ import json import os -from pkg_resources import parse_version from shutil import copy, rmtree from sqlite3 import OperationalError from tempfile import mkstemp @@ -33,6 +32,8 @@ Any, AnyStr, Dict, List, Optional, Set, TYPE_CHECKING, Tuple, Union ) +from packaging.version import parse as parse_version + from cylc.flow import LOG from cylc.flow.broadcast_report import get_broadcast_change_iter from cylc.flow.rundb import CylcWorkflowDAO diff --git a/cylc/flow/xtrigger_mgr.py b/cylc/flow/xtrigger_mgr.py index c89113641ef..50eede72a83 100644 --- a/cylc/flow/xtrigger_mgr.py +++ b/cylc/flow/xtrigger_mgr.py @@ -399,6 +399,7 @@ def get_xtrig_ctx( # External (clock xtrigger): convert offset to trigger_time. # Datetime cycling only. kwargs["trigger_time"] = itask.get_clock_trigger_time( + itask.point, ctx.func_kwargs["offset"] ) else: diff --git a/mypy.ini b/mypy.ini index 4c35cf1d53e..fe03c3bbe12 100644 --- a/mypy.ini +++ b/mypy.ini @@ -19,3 +19,7 @@ exclude= cylc/flow/etc/tutorial/.* # Suppress the following messages: # By default the bodies of untyped functions are not checked, consider using --check-untyped-defs disable_error_code = annotation-unchecked + +# For some reason, couldn't exclude this with the exclude directive above +[mypy-cylc.flow.data_messages_pb2] +ignore_errors = True diff --git a/setup.cfg b/setup.cfg index 6c17d2091ef..6b0ae3e1cc8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Programming Language :: Python :: 3 :: Only Programming Language :: Python :: Implementation :: CPython Topic :: Scientific/Engineering :: Atmospheric Science @@ -67,14 +68,13 @@ install_requires = graphene>=2.1,<3 # Note: can't pin jinja2 any higher than this until we give up on Cylc 7 back-compat jinja2==3.0.* - metomi-isodatetime==1!3.0.* + metomi-isodatetime>=1!3.0.0,<1!3.2.0 # Constrain protobuf version for compatible Scheduler-UIS comms across hosts - protobuf>=4.21.2,<4.22.0 + packaging + protobuf>=4.24.4,<4.25.0 psutil>=5.6.0 pyzmq>=22 - # https://github.com/pypa/setuptools/issues/3802 - setuptools>=49,!=67.* - importlib_metadata; python_version < "3.8" + importlib_metadata>=5.0; python_version < "3.12" urwid==2.* # unpinned transient dependencies used for type checking rx @@ -103,6 +103,7 @@ report-timings = pandas==1.* matplotlib tests = + aiosmtpd async_generator bandit>=1.7.0 coverage>=5.0.0,<7.3.1 @@ -116,18 +117,17 @@ tests = flake8-type-checking; python_version > "3.7" flake8>=3.0.0 mypy>=0.910 - pytest-asyncio>=0.17 + # https://github.com/pytest-dev/pytest-asyncio/issues/655 + pytest-asyncio>=0.17,!=0.22.0 pytest-cov>=2.8.0 pytest-xdist>=2 pytest-env>=0.6.2 - pytest-mock>=3.6.1 pytest>=6 testfixtures>=6.11.0 towncrier>=23 # Type annotation stubs # http://mypy-lang.blogspot.com/2021/05/the-upcoming-switch-to-modular-typeshed.html types-Jinja2>=0.1.3 - types-pkg_resources>=0.1.2 types-protobuf>=0.1.10 types-six>=0.1.6 typing-extensions>=4 @@ -141,7 +141,6 @@ all = %(main_loop-log_db)s %(main_loop-log_main_loop)s %(main_loop-log_memory)s - %(report-timings)s %(tests)s %(tutorials)s diff --git a/tests/conftest.py b/tests/conftest.py index 8f07a4f3441..8ed3e210fc6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,15 @@ from cylc.flow.parsec.validate import cylc_config_validate +@pytest.fixture(scope='module') +def mod_monkeypatch(): + """A module-scoped version of the monkeypatch fixture.""" + from _pytest.monkeypatch import MonkeyPatch + mpatch = MonkeyPatch() + yield mpatch + mpatch.undo() + + @pytest.fixture def mock_glbl_cfg(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """A Pytest fixture for fiddling global config values. diff --git a/tests/functional/cli/01-help.t b/tests/functional/cli/01-help.t index 3f78ada0dbf..935864e1873 100755 --- a/tests/functional/cli/01-help.t +++ b/tests/functional/cli/01-help.t @@ -19,7 +19,7 @@ . "$(dirname "$0")/test_header" # Number of tests depends on the number of 'cylc' commands. -set_test_number 26 +set_test_number 28 # Top help run_ok "${TEST_NAME_BASE}-0" cylc @@ -73,6 +73,10 @@ run_ok "${TEST_NAME_BASE}-version.stdout" \ cmp_ok "${TEST_NAME_BASE}-version.stdout" "${TEST_NAME_BASE}---version.stdout" cmp_ok "${TEST_NAME_BASE}-version.stdout" "${TEST_NAME_BASE}-V.stdout" +# Supplementary help +run_ok "${TEST_NAME_BASE}-all" cylc help all +run_ok "${TEST_NAME_BASE}-id" cylc help id + # Check "cylc version --long" output is correct. cylc version --long | head -n 1 > long1 WHICH="$(command -v cylc)" diff --git a/tests/functional/cylc-reinstall/00-simple.t b/tests/functional/cylc-reinstall/00-simple.t index ef880416bdf..d494f7dbc12 100644 --- a/tests/functional/cylc-reinstall/00-simple.t +++ b/tests/functional/cylc-reinstall/00-simple.t @@ -48,6 +48,7 @@ run_ok "${TEST_NAME}-reinstall" cylc reinstall "${RND_WORKFLOW_NAME}/run1" cmp_ok "${TEST_NAME}-reinstall.stdout" <<__OUT__ REINSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} Successfully reinstalled. +Done __OUT__ popd || exit 1 purge_rnd_workflow diff --git a/tests/functional/cylc-show/06-past-present-future/flow.cylc b/tests/functional/cylc-show/06-past-present-future/flow.cylc index 383ff86e0fb..5c8497d6a0a 100644 --- a/tests/functional/cylc-show/06-past-present-future/flow.cylc +++ b/tests/functional/cylc-show/06-past-present-future/flow.cylc @@ -20,6 +20,8 @@ cylc stop --now --max-polls=10 --interval=1 $CYLC_WORKFLOW_ID false else + # Allow time for c submission => running + sleep 2 cylc show "$CYLC_WORKFLOW_ID//1/b" >> $CYLC_WORKFLOW_RUN_DIR/show-b.txt cylc show "$CYLC_WORKFLOW_ID//1/c" >> $CYLC_WORKFLOW_RUN_DIR/show-c.txt cylc show "$CYLC_WORKFLOW_ID//1/d" >> $CYLC_WORKFLOW_RUN_DIR/show-d.txt diff --git a/tests/functional/events/09-task-event-mail.t b/tests/functional/events/09-task-event-mail.t index fb95c228a68..9f6195feb20 100755 --- a/tests/functional/events/09-task-event-mail.t +++ b/tests/functional/events/09-task-event-mail.t @@ -50,14 +50,16 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}" contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ -b'retry: 1/t1/01' -b'succeeded: 1/t1/02' -b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/' +retry: 1/t1/01 +succeeded: 1/t1/02 +see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ + + run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[1/t1/01 retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + grep -qPizo "Subject: \[1/t1/01 retry\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[1/t1/02 succeeded\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + grep -qPizo "Subject: \[1/t1/02 succeeded\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" purge mock_smtpd_kill diff --git a/tests/functional/events/18-workflow-event-mail.t b/tests/functional/events/18-workflow-event-mail.t index fcde6da0bd7..eae45962db0 100755 --- a/tests/functional/events/18-workflow-event-mail.t +++ b/tests/functional/events/18-workflow-event-mail.t @@ -49,11 +49,11 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}" contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ -b'event: startup' -b'message: workflow starting' -b'event: shutdown' -b'message: AUTOMATIC' -b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/' +event: startup +message: workflow starting +event: shutdown +message: AUTOMATIC +see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ purge diff --git a/tests/functional/events/29-task-event-mail-1.t b/tests/functional/events/29-task-event-mail-1.t index b6669ea8741..7ca6e76a7c2 100755 --- a/tests/functional/events/29-task-event-mail-1.t +++ b/tests/functional/events/29-task-event-mail-1.t @@ -38,11 +38,12 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "$WORKFLOW_NAME" contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ -b'retry: 1/t1/01' -b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/' +retry: 1/t1/01 +see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ + run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[1/t1/01 retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + grep -qPizo "Subject: \[1/t1/01 retry\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" purge mock_smtpd_kill diff --git a/tests/functional/events/30-task-event-mail-2.t b/tests/functional/events/30-task-event-mail-2.t index 6e8ff8a1e86..85f6b301654 100755 --- a/tests/functional/events/30-task-event-mail-2.t +++ b/tests/functional/events/30-task-event-mail-2.t @@ -50,27 +50,29 @@ workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}" contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ -b'retry: 1/t1/01' -b'retry: 1/t2/01' -b'retry: 1/t3/01' -b'retry: 1/t4/01' -b'retry: 1/t5/01' -b'retry: 1/t1/02' -b'retry: 1/t2/02' -b'retry: 1/t3/02' -b'retry: 1/t4/02' -b'retry: 1/t5/02' -b'failed: 1/t1/03' -b'failed: 1/t2/03' -b'failed: 1/t3/03' -b'failed: 1/t4/03' -b'failed: 1/t5/03' -b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/' +retry: 1/t1/01 +retry: 1/t2/01 +retry: 1/t3/01 +retry: 1/t4/01 +retry: 1/t5/01 +retry: 1/t1/02 +retry: 1/t2/02 +retry: 1/t3/02 +retry: 1/t4/02 +retry: 1/t5/02 +failed: 1/t1/03 +failed: 1/t2/03 +failed: 1/t3/03 +failed: 1/t4/03 +failed: 1/t5/03 +see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ + run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[. tasks retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + grep -qPizo "Subject: \[. tasks retry\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[. tasks failed\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" -purge + grep -qPizo "Subject: \[. tasks failed\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + + purge mock_smtpd_kill exit diff --git a/tests/functional/graphql/01-workflow.t b/tests/functional/graphql/01-workflow.t index b0f82997a0a..ae9044fe2c6 100755 --- a/tests/functional/graphql/01-workflow.t +++ b/tests/functional/graphql/01-workflow.t @@ -47,6 +47,7 @@ query { oldestActiveCyclePoint reloaded runMode + nEdgeDistance stateTotals workflowLogDir timeZoneInfo { @@ -96,6 +97,7 @@ cmp_json "${TEST_NAME}-out" "${TEST_NAME_BASE}-workflows.stdout" << __HERE__ "oldestActiveCyclePoint": "20210101T0000Z", "reloaded": false, "runMode": "live", + "nEdgeDistance": 1, "stateTotals": { "waiting": 1, "expired": 0, diff --git a/tests/functional/graphql/03-is-held-arg.t b/tests/functional/graphql/03-is-held-arg.t index 414f2842526..6603e1c5347 100755 --- a/tests/functional/graphql/03-is-held-arg.t +++ b/tests/functional/graphql/03-is-held-arg.t @@ -49,14 +49,14 @@ query { workflows { name isHeldTotal - taskProxies(isHeld: true) { + taskProxies(isHeld: true, graphDepth: 1) { id jobs { submittedTime startedTime } } - familyProxies(exids: [\"*/root\"], isHeld: true) { + familyProxies(exids: [\"*/root\"], isHeld: true, graphDepth: 1) { id } } diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header index 458a7dc908c..c4a07603126 100644 --- a/tests/functional/lib/bash/test_header +++ b/tests/functional/lib/bash/test_header @@ -807,10 +807,13 @@ mock_smtpd_init() { # Logic borrowed from Rose local SMTPD_LOG="${TEST_DIR}/smtpd.log" local SMTPD_HOST="localhost:${SMTPD_PORT}" # Set up fake SMTP server to catch outgoing mail & redirect to log: - python3 -u -m 'smtpd' -c 'DebuggingServer' -d -n "${SMTPD_HOST}" \ + python3 -u -m 'aiosmtpd' \ + --class aiosmtpd.handlers.Debugging stdout \ + --debug --nosetuid \ + --listen "${SMTPD_HOST}" \ 1>"${SMTPD_LOG}" 2>&1 & # Runs in background local SMTPD_PID="$!" - while ! grep -q 'DebuggingServer started' "${SMTPD_LOG}" 2>'/dev/null' + while ! grep -q 'is listening' "${SMTPD_LOG}" 2>'/dev/null' do if ps "${SMTPD_PID}" 1>/dev/null 2>&1; then sleep 1 # Still waiting for fake server to start diff --git a/tests/functional/modes/03-simulation.t b/tests/functional/modes/03-simulation.t new file mode 100644 index 00000000000..87a7ca37a9b --- /dev/null +++ b/tests/functional/modes/03-simulation.t @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Test that simulation mode runs, and reruns a failed task successfully +# when execution retry delays is configured. + +. "$(dirname "$0")/test_header" +set_test_number 2 + +install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" +run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" +workflow_run_ok "${TEST_NAME_BASE}-run" \ + cylc play \ + --no-detach \ + --mode=simulation \ + --reference-test "${WORKFLOW_NAME}" +purge +exit diff --git a/tests/functional/modes/03-simulation/flow.cylc b/tests/functional/modes/03-simulation/flow.cylc new file mode 100644 index 00000000000..49300212d39 --- /dev/null +++ b/tests/functional/modes/03-simulation/flow.cylc @@ -0,0 +1,16 @@ +[scheduler] + [[events]] + workflow timeout = PT30S + +[scheduling] + initial cycle point = 2359 + [[graph]] + R1 = get_observations + +[runtime] + [[get_observations]] + execution retry delays = PT2S + [[[simulation]]] + fail cycle points = all + fail try 1 only = True + diff --git a/tests/functional/modes/03-simulation/reference.log b/tests/functional/modes/03-simulation/reference.log new file mode 100644 index 00000000000..2d14bc201fb --- /dev/null +++ b/tests/functional/modes/03-simulation/reference.log @@ -0,0 +1,2 @@ +23590101T0000Z/get_observations -triggered off [] in flow 1 +23590101T0000Z/get_observations -triggered off [] in flow 1 diff --git a/tests/functional/n-window/01-past-present-future.t b/tests/functional/n-window/01-past-present-future.t new file mode 100644 index 00000000000..d5ed27a8085 --- /dev/null +++ b/tests/functional/n-window/01-past-present-future.t @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# Test window size using graphql and cylc-show for all tasks. + +. "$(dirname "$0")/test_header" + +set_test_number 7 + +install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" + +TEST_NAME="${TEST_NAME_BASE}-validate" +run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" + +TEST_NAME="${TEST_NAME_BASE}-run" +# 'a => b => c => d => e', 'a' sets window size to 2, 'c' uses cylc show on all. +workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}" + +TEST_NAME="${TEST_NAME_BASE}-show-a.past" +contains_ok "$WORKFLOW_RUN_DIR/show-a.txt" <<__END__ +state: succeeded +prerequisites: (None) +__END__ + +TEST_NAME="${TEST_NAME_BASE}-show-b.past" +contains_ok "$WORKFLOW_RUN_DIR/show-b.txt" <<__END__ +state: succeeded +prerequisites: (n/a for past tasks) +__END__ + +TEST_NAME="${TEST_NAME_BASE}-show-c.present" +contains_ok "${WORKFLOW_RUN_DIR}/show-c.txt" <<__END__ +prerequisites: ('-': not satisfied) + + 1/b succeeded +__END__ + +TEST_NAME="${TEST_NAME_BASE}-show-d.future" +contains_ok "${WORKFLOW_RUN_DIR}/show-d.txt" <<__END__ +state: waiting +prerequisites: ('-': not satisfied) + - 1/c succeeded +__END__ + +TEST_NAME="${TEST_NAME_BASE}-show-e.future" +contains_ok "${WORKFLOW_RUN_DIR}/show-e.txt" <<__END__ +state: waiting +prerequisites: ('-': not satisfied) + - 1/d succeeded +__END__ + +purge diff --git a/tests/functional/n-window/01-past-present-future/flow.cylc b/tests/functional/n-window/01-past-present-future/flow.cylc new file mode 100644 index 00000000000..a032274a41e --- /dev/null +++ b/tests/functional/n-window/01-past-present-future/flow.cylc @@ -0,0 +1,41 @@ +[scheduler] + allow implicit tasks = True + [[events]] + inactivity timeout = PT1M + abort on inactivity timeout = True +[scheduling] + [[graph]] + R1 = """ + a => b => c => d => e + """ +[runtime] + [[a]] + script = """ +set +e + +read -r -d '' gqlDoc <<_DOC_ +{"request_string": " +mutation { + setGraphWindowExtent ( + workflows: [\"${CYLC_WORKFLOW_ID}\"], + nEdgeDistance: 2) { + result + } +}", +"variables": null} +_DOC_ + +echo "${gqlDoc}" + +cylc client "$CYLC_WORKFLOW_ID" graphql < <(echo ${gqlDoc}) 2>/dev/null + +set -e +""" + [[c]] + script = """ +cylc show "$CYLC_WORKFLOW_ID//1/a" >> $CYLC_WORKFLOW_RUN_DIR/show-a.txt +cylc show "$CYLC_WORKFLOW_ID//1/b" >> $CYLC_WORKFLOW_RUN_DIR/show-b.txt +cylc show "$CYLC_WORKFLOW_ID//1/c" >> $CYLC_WORKFLOW_RUN_DIR/show-c.txt +cylc show "$CYLC_WORKFLOW_ID//1/d" >> $CYLC_WORKFLOW_RUN_DIR/show-d.txt +cylc show "$CYLC_WORKFLOW_ID//1/e" >> $CYLC_WORKFLOW_RUN_DIR/show-e.txt +""" diff --git a/tests/functional/n-window/02-big-window.t b/tests/functional/n-window/02-big-window.t new file mode 100644 index 00000000000..e6aa45fae24 --- /dev/null +++ b/tests/functional/n-window/02-big-window.t @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# Test large window size using graphql and find tasks in window. +# This is helpful with coverage by using most the no-rewalk mechanics. + +. "$(dirname "$0")/test_header" + +set_test_number 5 + +install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" + +TEST_NAME="${TEST_NAME_BASE}-validate" +run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" + +TEST_NAME="${TEST_NAME_BASE}-run" +# 'a => b => . . . f => g => h', 'a' sets window size to 5, +# 'b => i => j => f', 'c' finds 'a', 'j', 'h' +workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}" + +TEST_NAME="${TEST_NAME_BASE}-show-a.past" +contains_ok "$WORKFLOW_RUN_DIR/show-a.txt" <<__END__ +state: succeeded +prerequisites: (None) +__END__ + +TEST_NAME="${TEST_NAME_BASE}-show-j.parallel" +contains_ok "${WORKFLOW_RUN_DIR}/show-j.txt" <<__END__ +state: waiting +prerequisites: ('-': not satisfied) + - 1/i succeeded +__END__ + +TEST_NAME="${TEST_NAME_BASE}-show-h.future" +contains_ok "${WORKFLOW_RUN_DIR}/show-h.txt" <<__END__ +state: waiting +prerequisites: ('-': not satisfied) + - 1/g succeeded +__END__ + +purge diff --git a/tests/functional/n-window/02-big-window/flow.cylc b/tests/functional/n-window/02-big-window/flow.cylc new file mode 100644 index 00000000000..09e4d8181fc --- /dev/null +++ b/tests/functional/n-window/02-big-window/flow.cylc @@ -0,0 +1,52 @@ +[scheduler] + allow implicit tasks = True + [[events]] + inactivity timeout = PT1M + abort on inactivity timeout = True +[scheduling] + [[graph]] + R1 = """ + a => b => c => d => e => f => g => h + b => i => j => f + """ +[runtime] + [[a]] + script = """ +set +e + +read -r -d '' gqlDoc <<_DOC_ +{"request_string": " +mutation { + setGraphWindowExtent ( + workflows: [\"${CYLC_WORKFLOW_ID}\"], + nEdgeDistance: 5) { + result + } +}", +"variables": null} +_DOC_ + +echo "${gqlDoc}" + +cylc client "$CYLC_WORKFLOW_ID" graphql < <(echo ${gqlDoc}) 2>/dev/null + +set -e +""" + [[c]] + script = """ +cylc show "$CYLC_WORKFLOW_ID//1/a" >> $CYLC_WORKFLOW_RUN_DIR/show-a.txt +cylc show "$CYLC_WORKFLOW_ID//1/j" >> $CYLC_WORKFLOW_RUN_DIR/show-j.txt +cylc show "$CYLC_WORKFLOW_ID//1/h" >> $CYLC_WORKFLOW_RUN_DIR/show-h.txt +""" + + [[i]] + script = """ +# Slow 2nd branch down +sleep 5 +""" + + [[f]] + script = """ +# test re-trigger of old point +cylc trigger "$CYLC_WORKFLOW_ID//1/b" +""" diff --git a/tests/functional/n-window/test_header b/tests/functional/n-window/test_header new file mode 120000 index 00000000000..90bd5a36f92 --- /dev/null +++ b/tests/functional/n-window/test_header @@ -0,0 +1 @@ +../lib/bash/test_header \ No newline at end of file diff --git a/tests/integration/README.md b/tests/integration/README.md index 71bd6bfb922..1a776232a2b 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -12,7 +12,9 @@ $ pytest tests/i --dist=no -n0 # turn off xdist (allows --pdb etc) ## What Are Integration Tests -These tests are intended to test the interaction of different modules. +Integration tests aren't end-to-end tests. They focus on targeted interactions +of multiple modules and may do a bit of monkeypatching to achieve that result. + With Cylc this typically involves running workflows. The general approach is: @@ -21,8 +23,34 @@ The general approach is: 2) Put it in a funny state. 3) Test how components interract to handle this state. -Integration tests aren't end-to-end tests, they focus on targeted interactions -and behaviour and may do a bit of monkeypatching to achieve that result. +I.e., the integration test framework runs the scheduler. The only thing it's +really cutting out is the CLI. + +You can do everything, up to and including reference tests with it if so +inclined, although that would really be a functional test implemented in Python: + +async with run(schd) as log: + # run the workflow with a timeout of 60 seconds + await asyncio.sleep(60) +assert reftest(log) == ''' +1/b triggered off [1/a] +1/c triggered off [1/b] +''' + +For a more integration'y approach to reftests we can do something like this +which is essentially just another way of getting the "triggered off" information +without having to run the main loop and bring race conditions into play: + +async with start(schd): + assert set(schd.pool.get_tasks()) == {'1/a'} + + # setting a:succeeded should spawn b + schd.command_reset('1/a', 'succeeded') + assert set(schd.pool.get_tasks()) == {'1/b'} + + # setting b:x should spawn c + schd.command_reset('1/b', 'x') + assert set(schd.pool.get_tasks()) == {'1/b', '1/c'} ## Guidelines diff --git a/tests/integration/test_increment_graph_window.py b/tests/integration/test_increment_graph_window.py new file mode 100644 index 00000000000..bd418a8bad5 --- /dev/null +++ b/tests/integration/test_increment_graph_window.py @@ -0,0 +1,410 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from contextlib import suppress + +from cylc.flow.cycling.integer import IntegerPoint +from cylc.flow.data_store_mgr import ( + TASK_PROXIES, +) +from cylc.flow.id import Tokens + + +def increment_graph_window(schd, task): + """Increment the graph window about the active task.""" + tokens = schd.tokens.duplicate(cycle='1', task=task) + schd.data_store_mgr.increment_graph_window( + tokens, + IntegerPoint('1'), + {1}, + is_manual_submit=False, + ) + + +def get_deltas(schd): + """Return the ids and graph-window values in the delta store. + + Note, call before get_n_window as this clears the delta store. + + Returns: + (added, updated, pruned) + + """ + # populate added deltas + schd.data_store_mgr.gather_delta_elements( + schd.data_store_mgr.added, + 'added', + ) + # populate pruned deltas + schd.data_store_mgr.prune_data_store() + # Run depth finder + schd.data_store_mgr.window_depth_finder() + # populate updated deltas + schd.data_store_mgr.gather_delta_elements( + schd.data_store_mgr.updated, + 'updated', + ) + return ( + { + # added + Tokens(tb_task_proxy.id)['task']: tb_task_proxy.graph_depth + for tb_task_proxy in schd.data_store_mgr.deltas[TASK_PROXIES].added + }, + { + # updated + Tokens(tb_task_proxy.id)['task']: tb_task_proxy.graph_depth + for tb_task_proxy in schd.data_store_mgr.deltas[TASK_PROXIES].updated + # only include those updated nodes whose depths have been set + if 'graph_depth' in { + sub_field.name + for sub_field, _ in tb_task_proxy.ListFields() + } + }, + { + # pruned + Tokens(id_)['task'] + for id_ in schd.data_store_mgr.deltas[TASK_PROXIES].pruned + }, + ) + + +async def get_n_window(schd): + """Read out the graph window of the workflow.""" + await schd.update_data_structure() + data = schd.data_store_mgr.data[schd.data_store_mgr.workflow_id] + return { + t.name: t.graph_depth + for t in data[TASK_PROXIES].values() + } + + +async def complete_task(schd, task): + """Mark a task as completed.""" + schd.data_store_mgr.remove_pool_node(task, IntegerPoint('1')) + + +def add_task(schd, task): + """Add a waiting task to the pool.""" + schd.data_store_mgr.add_pool_node(task, IntegerPoint('1')) + + +def get_graph_walk_cache(schd): + """Return the head task names of cached graph walks.""" + # prune graph walk cache + schd.data_store_mgr.prune_data_store() + # fetch the cached walks + n_window_node_walks = sorted( + Tokens(task_id)['task'] + for task_id in schd.data_store_mgr.n_window_node_walks + ) + n_window_completed_walks = sorted( + Tokens(task_id)['task'] + for task_id in schd.data_store_mgr.n_window_completed_walks + ) + # the IDs in set and keys of dict are only the same at n<2 window. + assert n_window_node_walks == n_window_completed_walks + return n_window_completed_walks + + +async def test_increment_graph_window_blink(flow, scheduler, start): + """Test with a task which drifts in and out of the n-window. + + This workflow presents a fiendish challenge for the graph window algorithm. + + The test runs in the n=3 window and simulates running each task in the + chain a - s one by one. The task "blink" is dependent on multiple tasks + in the chain awkwardly spaced so that the "blink" task routinely + disappears from the n-window, only to re-appear again later. + + The expansion of the window around the "blink" task is difficult to get + right as it can be influenced by caches from previous graph walks. + """ + id_ = flow({ + 'scheduler': { + 'allow implicit tasks': 'True', + }, + 'scheduling': { + 'cycling mode': 'integer', + 'initial cycle point': '1', + 'graph': { + 'R1': ''' + # the "abdef" chain of tasks which run one after another + a => b => c => d => e => f => g => h => i => j => k => l => + m => n => o => p => q => r => s + + # these dependencies cause "blink" to disappear and + # reappear at set intervals + a => blink + g => blink + m => blink + s => blink + ''', + } + } + }) + schd = scheduler(id_) + + # the tasks traversed via the "blink" task when... + blink = { + 1: { + # the blink task is n=1 + 'blink': 1, + 'a': 2, + 'g': 2, + 'm': 2, + 's': 2, + 'b': 3, + 'f': 3, + 'h': 3, + 'l': 3, + 'n': 3, + 'r': 3, + }, + 2: { + # the blink task is n=2 + 'blink': 2, + 'a': 3, + 'g': 3, + 'm': 3, + 's': 3, + }, + 3: { + # the blink task is n=3 + 'blink': 3, + }, + 4: { + # the blink task is n=4 + }, + } + + def advance(): + """Advance to the next task in the workflow. + + This works its way down the chain of tasks between "a" and "s" + inclusive, yielding what the n-window should look like for this + workflow at each step. + + Yields: + tuple - (previous_task, active_task, n_window) + + previous_task: + The task which has just "succeeded". + active_task: + The task which is about to run. + n_window: + Dictionary of {task_name: graph_depth} for the n=3 window. + + """ + # the initial window on startup (minus the nodes traversed via "blink") + window = { + 'a': 0, + 'b': 1, + 'c': 2, + 'd': 3, + } + # the tasks we will run in order + letters = 'abcdefghijklmnopqrs' + # the graph-depth of the "blink" task at each stage of the workflow + blink_distances = [1] + [*range(2, 5), *range(3, 0, -1)] * 3 + + for ind, blink_distance in zip(range(len(letters)), blink_distances): + previous_task = letters[ind - 1] if ind > 0 else None + active_task = letters[ind] + yield ( + previous_task, + active_task, + { + # the tasks traversed via the "blink" task + **blink[blink_distance], + # the tasks in the main "abcdefg" chain + **{key: abs(value) for key, value in window.items()}, + } + ) + + # move each task in the "abcdef" chain down one + window = {key: value - 1 for key, value in window.items()} + # add the n=3 task in the "abcdef" chain into the window + with suppress(IndexError): + window[letters[ind + 4]] = 3 + # pull out anything which is not supposed to be in the n=3 window + window = { + key: value + for key, value in window.items() + if abs(value) < 4 + } + + async with start(schd): + schd.data_store_mgr.set_graph_window_extent(3) + await schd.update_data_structure() + + previous_n_window = {} + for previous_task, active_task, expected_n_window in advance(): + # mark the previous task as completed + await complete_task(schd, previous_task) + # add the next task to the pool + add_task(schd, active_task) + # run the graph window algorithm + increment_graph_window(schd, active_task) + # get the deltas which increment_graph_window created + added, updated, pruned = get_deltas(schd) + + # compare the n-window in the store to what we were expecting + n_window = await get_n_window(schd) + assert n_window == expected_n_window + + # compare the deltas to what we were expecting + if active_task != 'a': + # skip the first task as this is complicated by startup logic + assert added == { + key: value + for key, value in expected_n_window.items() + if key not in previous_n_window + } + # Skip added as depth isn't updated + # (the manager only updates those that need it) + assert updated == { + key: value + for key, value in expected_n_window.items() + if key not in added + } + assert pruned == { + key + for key in previous_n_window + if key not in expected_n_window + } + + previous_n_window = n_window + + +async def test_window_resize_rewalk(flow, scheduler, start): + """The window resize method should wipe and rebuild the n-window.""" + id_ = flow({ + 'scheduler': { + 'allow implicit tasks': 'true', + }, + 'scheduling': { + 'graph': { + 'R1': 'a => b => c => d => e => f => g' + } + }, + }) + schd = scheduler(id_) + async with start(schd): + # start with an empty pool + schd.pool.remove(schd.pool.get_tasks()[0]) + + # the n-window should be empty + assert await get_n_window(schd) == {} + + # expand the window around 1/d + add_task(schd, 'd') + increment_graph_window(schd, 'd') + + # set the graph window to n=3 + schd.data_store_mgr.set_graph_window_extent(3) + assert set(await get_n_window(schd)) == { + 'a', 'b', 'c', 'd', 'e', 'f', 'g' + } + + # set the graph window to n=1 + schd.data_store_mgr.set_graph_window_extent(1) + schd.data_store_mgr.window_resize_rewalk() + assert set(await get_n_window(schd)) == { + 'c', 'd', 'e' + } + + # set the graph window to n=2 + schd.data_store_mgr.set_graph_window_extent(2) + schd.data_store_mgr.window_resize_rewalk() + assert set(await get_n_window(schd)) == { + 'b', 'c', 'd', 'e', 'f' + } + + +async def test_cache_pruning(flow, scheduler, start): + """It should remove graph walks from the cache when no longer needed. + + The algorithm caches graph walks for efficiency. This test is designed to + ensure we don't introduce a memory leak by failing to clear cached walks + at the correct point. + """ + id_ = flow({ + 'scheduler': { + 'allow implicit tasks': 'True', + }, + 'scheduling': { + 'graph': { + 'R1': ''' + # a chain of tasks + a => b1 & b2 => c => d1 & d2 => e => f + # force "a" to drift into an out of the window + a => c + a => e + ''' + } + }, + }) + schd = scheduler(id_) + async with start(schd): + schd.data_store_mgr.set_graph_window_extent(1) + + # work through this workflow, step by step checking the cached items... + + # active: a + add_task(schd, 'a') + increment_graph_window(schd, 'a') + assert get_graph_walk_cache(schd) == ['a'] + + # active: b1, b2 + await complete_task(schd, 'a') + add_task(schd, 'b1') + add_task(schd, 'b2') + increment_graph_window(schd, 'b1') + increment_graph_window(schd, 'b2') + assert get_graph_walk_cache(schd) == ['a', 'b1', 'b2'] + + # active: c + await complete_task(schd, 'b1') + await complete_task(schd, 'b2') + add_task(schd, 'c') + increment_graph_window(schd, 'c') + assert get_graph_walk_cache(schd) == ['a', 'b1', 'b2', 'c'] + + # active: d1, d2 + await complete_task(schd, 'c') + add_task(schd, 'd1') + add_task(schd, 'd2') + increment_graph_window(schd, 'd1') + increment_graph_window(schd, 'd2') + assert get_graph_walk_cache(schd) == ['c', 'd1', 'd2'] + + # active: e + await complete_task(schd, 'd1') + await complete_task(schd, 'd2') + add_task(schd, 'e') + increment_graph_window(schd, 'e') + assert get_graph_walk_cache(schd) == ['d1', 'd2', 'e'] + + # active: f + await complete_task(schd, 'e') + add_task(schd, 'f') + increment_graph_window(schd, 'f') + assert get_graph_walk_cache(schd) == ['e', 'f'] + + # active: None + await complete_task(schd, 'f') + increment_graph_window(schd, 'f') + assert get_graph_walk_cache(schd) == [] diff --git a/tests/integration/test_install.py b/tests/integration/test_install.py index cbac55c5361..f681c5a2a01 100644 --- a/tests/integration/test_install.py +++ b/tests/integration/test_install.py @@ -162,7 +162,7 @@ def test_install_gets_back_compat_mode_for_plugins( class failIfDeprecated: """A fake Cylc Plugin entry point""" @staticmethod - def resolve(): + def load(): return failIfDeprecated.raiser @staticmethod diff --git a/tests/integration/test_reinstall.py b/tests/integration/test_reinstall.py index 3dadc4b7df0..1b965f18b8e 100644 --- a/tests/integration/test_reinstall.py +++ b/tests/integration/test_reinstall.py @@ -19,6 +19,7 @@ from pathlib import Path from types import SimpleNamespace from uuid import uuid1 +from functools import partial import pytest @@ -37,7 +38,7 @@ from cylc.flow.workflow_files import ( WorkflowFiles, ) - +from cylc.flow.network.multi import call_multi ReInstallOptions = Options(reinstall_gop()) @@ -90,14 +91,14 @@ def one_run(one_src, test_dir, run_dir): ) -def test_rejects_random_workflows(one): +async def test_rejects_random_workflows(one, one_run): """It should only work with workflows installed by cylc install.""" with pytest.raises(WorkflowFilesError) as exc_ctx: - reinstall_cli(opts=ReInstallOptions(), args=one.workflow) + await reinstall_cli(opts=ReInstallOptions(), workflow_id=one.workflow) assert 'was not installed with cylc install' in str(exc_ctx.value) -def test_invalid_source_dir(one_src, one_run): +async def test_invalid_source_dir(one_src, one_run): """It should detect & fail for an invalid source symlink""" source_link = Path( one_run.path, @@ -108,22 +109,22 @@ def test_invalid_source_dir(one_src, one_run): source_link.symlink_to(one_src.path / 'flow.cylc') with pytest.raises(WorkflowFilesError) as exc_ctx: - reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert 'Workflow source dir is not accessible' in str(exc_ctx.value) -def test_no_changes_needed(one_src, one_run, capsys, interactive): +async def test_no_changes_needed(one_src, one_run, capsys, interactive): """It should not reinstall if no changes are needed. This is not a hard requirement, in practice rsync output may differ from expectation so this is a nice-to-have, not expected to work 100% of the time. """ - assert not reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + assert not await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert 'up to date with' in capsys.readouterr().out -def test_non_interactive(one_src, one_run, capsys, capcall, non_interactive): +async def test_non_interactive(one_src, one_run, capsys, capcall, non_interactive): """It should not perform a dry-run or prompt in non-interactive mode.""" # capture reinstall calls reinstall_calls = capcall( @@ -133,13 +134,13 @@ def test_non_interactive(one_src, one_run, capsys, capcall, non_interactive): # give it something to reinstall (one_src.path / 'a').touch() # reinstall - assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) # only one rsync call should have been made (i.e. no --dry-run) assert len(reinstall_calls) == 1 assert 'Successfully reinstalled' in capsys.readouterr().out -def test_interactive( +async def test_interactive( one_src, one_run, capsys, @@ -161,7 +162,7 @@ def test_interactive( 'cylc.flow.scripts.reinstall._input', lambda x: 'n' ) - assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id) is False + assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) is False # only one rsync call should have been made (i.e. the --dry-run) assert [call[1].get('dry_run') for call in reinstall_calls] == [True] @@ -173,7 +174,7 @@ def test_interactive( 'cylc.flow.scripts.reinstall._input', lambda x: 'y' ) - assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) # two rsync calls should have been made (i.e. the --dry-run and the real) assert [call[1].get('dry_run') for call in reinstall_calls] == [ @@ -182,7 +183,7 @@ def test_interactive( assert 'Successfully reinstalled' in capsys.readouterr().out -def test_workflow_running( +async def test_workflow_running( one_src, one_run, monkeypatch, @@ -194,7 +195,7 @@ def test_workflow_running( reload_message = f'Run "cylc reload {one_run.id}"' # reinstall with a stopped workflow (reload message shouldn't show) - assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert reload_message not in capsys.readouterr().out # reinstall with a running workflow (reload message should show) @@ -203,11 +204,11 @@ def test_workflow_running( 'cylc.flow.scripts.reinstall.load_contact_file', lambda x: None, ) - assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert reload_message in capsys.readouterr().out -def test_rsync_stuff(one_src, one_run, capsys, non_interactive): +async def test_rsync_stuff(one_src, one_run, capsys, non_interactive): """Make sure rsync is working correctly.""" # src contains files: a, b (one_src.path / 'a').touch() @@ -219,7 +220,7 @@ def test_rsync_stuff(one_src, one_run, capsys, non_interactive): (one_run.path / 'b').touch() (one_run.path / 'c').touch() - reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) # a should have been left assert (one_run.path / 'a').exists() @@ -231,7 +232,7 @@ def test_rsync_stuff(one_src, one_run, capsys, non_interactive): assert not (one_run.path / 'c').exists() -def test_rose_warning(one_src, one_run, capsys, interactive, monkeypatch): +async def test_rose_warning(one_src, one_run, capsys, interactive, monkeypatch): """It should warn that Rose installed files will be deleted. See https://github.com/cylc/cylc-rose/issues/149 @@ -249,16 +250,16 @@ def test_rose_warning(one_src, one_run, capsys, interactive, monkeypatch): (one_src.path / 'a').touch() # give it something to install # reinstall (with rose-suite.conf file) - reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert rose_message in capsys.readouterr().err # reinstall (no rose-suite.conf file) (one_src.path / 'rose-suite.conf').unlink() - reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert rose_message not in capsys.readouterr().err -def test_keyboard_interrupt( +async def test_keyboard_interrupt( one_src, one_run, interactive, @@ -279,11 +280,11 @@ def raise_keyboard_interrupt(): raise_keyboard_interrupt, ) - reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert 'Reinstall canceled, no changes made' in capsys.readouterr().out -def test_rsync_fail(one_src, one_run, mock_glbl_cfg, non_interactive): +async def test_rsync_fail(one_src, one_run, mock_glbl_cfg, non_interactive): """It should raise an error on rsync failure.""" mock_glbl_cfg( 'cylc.flow.install.glbl_cfg', @@ -296,5 +297,5 @@ def test_rsync_fail(one_src, one_run, mock_glbl_cfg, non_interactive): (one_src.path / 'a').touch() # give it something to install with pytest.raises(WorkflowFilesError) as exc_ctx: - reinstall_cli(opts=ReInstallOptions(), args=one_run.id) + await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) assert 'An error occurred reinstalling' in str(exc_ctx.value) diff --git a/tests/integration/test_simulation.py b/tests/integration/test_simulation.py new file mode 100644 index 00000000000..e0ada2c3e49 --- /dev/null +++ b/tests/integration/test_simulation.py @@ -0,0 +1,125 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import pytest +from queue import Queue +from types import SimpleNamespace + +from cylc.flow.cycling.iso8601 import ISO8601Point +from cylc.flow.simulation import sim_time_check + + +def get_msg_queue_item(queue, id_): + for item in queue.queue: + if id_ in str(item.job_id): + return item + + +@pytest.fixture(scope='module') +async def sim_time_check_setup( + mod_flow, mod_scheduler, mod_start, mod_one_conf +): + schd = mod_scheduler(mod_flow({ + 'scheduler': {'cycle point format': '%Y'}, + 'scheduling': { + 'initial cycle point': '1066', + 'graph': { + 'R1': 'one & fail_all & fast_forward' + } + }, + 'runtime': { + 'one': {}, + 'fail_all': { + 'simulation': {'fail cycle points': 'all'}, + 'outputs': {'foo': 'bar'} + }, + # This task ought not be finished quickly, but for the speed up + 'fast_forward': { + 'execution time limit': 'PT1M', + 'simulation': {'speedup factor': 2} + } + } + })) + msg_q = Queue() + async with mod_start(schd): + itasks = schd.pool.get_tasks() + for i in itasks: + i.try_timers = {'execution-retry': SimpleNamespace(num=0)} + yield schd, itasks, msg_q + + +def test_false_if_not_running(sim_time_check_setup, monkeypatch): + schd, itasks, msg_q = sim_time_check_setup + + # False if task status not running: + assert sim_time_check(msg_q, itasks) is False + + +def test_sim_time_check_sets_started_time(sim_time_check_setup): + """But sim_time_check still returns False""" + schd, _, msg_q = sim_time_check_setup + one_1066 = schd.pool.get_task(ISO8601Point('1066'), 'one') + one_1066.state.status = 'running' + assert one_1066.summary['started_time'] is None + assert sim_time_check(msg_q, [one_1066]) is False + assert one_1066.summary['started_time'] is not None + + +def test_task_finishes(sim_time_check_setup, monkeypatch): + """...and an appropriate message sent. + + Checks all possible outcomes in sim_time_check where elapsed time is + greater than the simulation time. + + Does NOT check every possible cause on an outcome - this is done + in unit tests. + """ + schd, _, msg_q = sim_time_check_setup + monkeypatch.setattr('cylc.flow.simulation.time', lambda: 0) + + # Setup a task to fail + fail_all_1066 = schd.pool.get_task(ISO8601Point('1066'), 'fail_all') + fail_all_1066.state.status = 'running' + fail_all_1066.try_timers = {'execution-retry': SimpleNamespace(num=0)} + + # Before simulation time is up: + assert sim_time_check(msg_q, [fail_all_1066]) is False + + # After simulation time is up: + monkeypatch.setattr('cylc.flow.simulation.time', lambda: 12) + assert sim_time_check(msg_q, [fail_all_1066]) is True + assert get_msg_queue_item(msg_q, '1066/fail_all').message == 'failed' + + # Succeeds and records messages for all outputs: + fail_all_1066.try_timers = {'execution-retry': SimpleNamespace(num=1)} + msg_q = Queue() + assert sim_time_check(msg_q, [fail_all_1066]) is True + assert sorted(i.message for i in msg_q.queue) == ['bar', 'succeeded'] + + +def test_task_sped_up(sim_time_check_setup, monkeypatch): + """Task will speed up by a factor set in config.""" + schd, _, msg_q = sim_time_check_setup + fast_forward_1066 = schd.pool.get_task( + ISO8601Point('1066'), 'fast_forward') + fast_forward_1066.state.status = 'running' + + monkeypatch.setattr('cylc.flow.simulation.time', lambda: 0) + assert sim_time_check(msg_q, [fast_forward_1066]) is False + monkeypatch.setattr('cylc.flow.simulation.time', lambda: 29) + assert sim_time_check(msg_q, [fast_forward_1066]) is False + monkeypatch.setattr('cylc.flow.simulation.time', lambda: 31) + assert sim_time_check(msg_q, [fast_forward_1066]) is True diff --git a/tests/integration/test_xtrigger_mgr.py b/tests/integration/test_xtrigger_mgr.py new file mode 100644 index 00000000000..07abbdac24d --- /dev/null +++ b/tests/integration/test_xtrigger_mgr.py @@ -0,0 +1,67 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Tests for the behaviour of xtrigger manager. +""" + + +async def test_2_xtriggers(flow, start, scheduler, monkeypatch): + """Test that if an itask has 2 wall_clock triggers with different + offsets that xtrigger manager gets both of them. + + https://github.com/cylc/cylc-flow/issues/5783 + + n.b. Clock 3 exists to check the memoization path is followed, + and causing this test to give greater coverage. + """ + task_point = 1588636800 # 2020-05-05 + ten_years_ahead = 1904169600 # 2030-05-05 + monkeypatch.setattr( + 'cylc.flow.xtriggers.wall_clock.time', + lambda: ten_years_ahead - 1 + ) + id_ = flow({ + 'scheduler': { + 'allow implicit tasks': True + }, + 'scheduling': { + 'initial cycle point': '2020-05-05', + 'xtriggers': { + 'clock_1': 'wall_clock()', + 'clock_2': 'wall_clock(offset=P10Y)', + 'clock_3': 'wall_clock(offset=P10Y)', + }, + 'graph': { + 'R1': '@clock_1 & @clock_2 & @clock_3 => foo' + } + } + }) + schd = scheduler(id_) + async with start(schd): + foo_proxy = schd.pool.get_tasks()[0] + clock_1_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_1') + clock_2_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_2') + clock_3_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_2') + + assert clock_1_ctx.func_kwargs['trigger_time'] == task_point + assert clock_2_ctx.func_kwargs['trigger_time'] == ten_years_ahead + assert clock_3_ctx.func_kwargs['trigger_time'] == ten_years_ahead + + schd.xtrigger_mgr.call_xtriggers_async(foo_proxy) + assert foo_proxy.state.xtriggers == { + 'clock_1': True, + 'clock_2': False, + 'clock_3': False, + } diff --git a/tests/integration/tui/__init__.py b/tests/integration/tui/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/tui/conftest.py b/tests/integration/tui/conftest.py new file mode 100644 index 00000000000..5b9ddcae61b --- /dev/null +++ b/tests/integration/tui/conftest.py @@ -0,0 +1,304 @@ +from contextlib import contextmanager +from difflib import unified_diff +import os +from pathlib import Path +import re +from time import sleep +from uuid import uuid1 + +import pytest +from urwid import html_fragment + +from cylc.flow.id import Tokens +from cylc.flow.tui.app import TuiApp +from cylc.flow.tui.overlay import _get_display_id + + +SCREENSHOT_DIR = Path(__file__).parent / 'screenshots' + + +def configure_screenshot(v_term_size): + """Configure Urwid HTML screenshots.""" + screen = html_fragment.HtmlGenerator() + screen.set_terminal_properties(256) + screen.register_palette(TuiApp.palette) + html_fragment.screenshot_init( + [tuple(map(int, v_term_size.split(',')))], + [] + ) + return screen, html_fragment + + +def format_test_failure(expected, got, description): + """Return HTML to represent a screenshot test failure. + + Args: + expected: + HTML fragment for the expected screenshot. + got: + HTML fragment for the test screenshot. + description: + Test description. + + """ + diff = '\n'.join(unified_diff( + expected.splitlines(), + got.splitlines(), + fromfile='expected', + tofile='got', + )) + return f''' + + +

{description}

+ + + + + + + + + +
ExpectedGot
{expected}{got}
+
+

Diff:

+ 
{ diff }
+ + ''' + + +class RaikuraSession: + """Convenience class for accessing Raikura functionality.""" + + def __init__(self, app, html_fragment, test_dir, test_name): + self.app = app + self.html_fragment = html_fragment + self.test_dir = test_dir + self.test_name = test_name + + def user_input(self, *keys): + """Simulate a user pressing keys. + + Each "key" is a keyboard button e.g. "x" or "enter". + + If you provide more than one key, each one will be pressed, one + after another. + + You can combine keys in a single string, e.g. "ctrl d". + """ + return self.app.loop.process_input(keys) + + def compare_screenshot( + self, + name, + description, + retries=3, + delay=0.1, + force_update=True, + ): + """Take a screenshot and compare it to one taken previously. + + To update the screenshot, set the environment variable + "CYLC_UPDATE_SCREENSHOTS" to "true". + + Note, if the comparison fails, "force_update" is called and the + test is retried. + + Arguments: + name: + The name to use for the screenshot, this is used in the + filename for the generated HTML fragment. + description: + A description of the test to be used on test failure. + retries: + The maximum number of retries for this test before failing. + delay: + The delay between retries. This helps overcome timing issues + with data provision. + + Raises: + Exception: + If the screenshot does not match the reference. + + """ + filename = SCREENSHOT_DIR / f'{self.test_name}.{name}.html' + + exc = None + for _try in range(retries): + # load the expected result + expected = '' + if filename.exists(): + with open(filename, 'r') as expected_file: + expected = expected_file.read() + # update to pick up latest data + if force_update: + self.force_update() + # force urwid to draw the screen + # (the main loop isn't runing so this doesn't happen automatically) + self.app.loop.draw_screen() + # take a screenshot + screenshot = self.html_fragment.screenshot_collect()[-1] + + try: + if expected != screenshot: + # screenshot does not match + # => write an html file with the visual diff + out = self.test_dir / filename.name + with open(out, 'w+') as out_file: + out_file.write( + format_test_failure( + expected, + screenshot, + description, + ) + ) + raise Exception( + 'Screenshot differs:' + '\n* Set "CYLC_UPDATE_SCREENSHOTS=true" to update' + f'\n* To debug see: file:////{out}' + ) + + break + except Exception as exc_: + exc = exc_ + # wait a while to allow the updater to do its job + sleep(delay) + else: + if os.environ.get('CYLC_UPDATE_SCREENSHOTS', '').lower() == 'true': + with open(filename, 'w+') as expected_file: + expected_file.write(screenshot) + else: + raise exc + + def force_update(self): + """Run Tui's update method. + + This is done automatically by compare_screenshot but you may want to + call it in a test, e.g. before pressing navigation keys. + + With Raikura, the Tui event loop is not running so the data is never + refreshed. + + You do NOT need to call this method for key presses, but you do need to + call this if the data has changed (e.g. if you've changed a task state) + OR if you've changed any filters (because filters are handled by the + update code). + """ + # flush any prior updates + self.app.get_update() + # wait for the next update + while not self.app.update(): + pass + + def wait_until_loaded(self, *ids, retries=20): + """Wait until the given ID appears in the Tui tree, then expand them. + + Useful for waiting whilst Tui loads a workflow. + + Note, this is a blocking wait with no timeout! + """ + ids = self.app.wait_until_loaded(*ids, retries=retries) + if ids: + self.compare_screenshot( + f'fail-{uuid1()}', + ( + 'Requested nodes did not appear in Tui after' + f' {retries} retries: ' + + ', '.join(ids) + ), + 1, + ) + + +@pytest.fixture +def raikura(test_dir, request, monkeypatch): + """Visual regression test framework for Urwid apps. + + Like Cypress but for Tui so named after a NZ island with lots of Tuis. + + When called this yields a RaikuraSession object loaded with test + utilities. All tests have default retries to avoid flaky tests. + + Similar to the "start" fixture, which starts a Scheduler without running + the main loop, raikura starts Tui without running the main loop. + + Arguments: + workflow_id: + The "WORKFLOW" argument of the "cylc tui" command line. + size: + The virtual terminal size for screenshots as a comma + separated string e.g. "80,50" for 80 cols wide by 50 rows tall. + + Returns: + A RaikuraSession context manager which provides useful utilities for + testing. + + """ + return _raikura(test_dir, request, monkeypatch) + + +@pytest.fixture +def mod_raikura(test_dir, request, monkeypatch): + """Same as raikura but configured to view module-scoped workflows. + + Note: This is *not* a module-scoped fixture (no need, creating Tui sessions + is not especially slow), it is configured to display module-scoped + "scheduler" fixtures (which may be more expensive to create/destroy). + """ + return _raikura(test_dir.parent, request, monkeypatch) + + +def _raikura(test_dir, request, monkeypatch): + # make the workflow and scan update intervals match (more reliable) + # and speed things up a little whilst we're at it + monkeypatch.setattr( + 'cylc.flow.tui.updater.Updater.BASE_UPDATE_INTERVAL', + 0.1, + ) + monkeypatch.setattr( + 'cylc.flow.tui.updater.Updater.BASE_SCAN_INTERVAL', + 0.1, + ) + + # the user name and the prefix of workflow IDs are both variable + # so we patch the render functions to make test output stable + def get_display_id(id_): + tokens = Tokens(id_) + return _get_display_id( + tokens.duplicate( + user='cylc', + workflow=tokens.get('workflow', '').rsplit('/', 1)[-1], + ).id + ) + monkeypatch.setattr('cylc.flow.tui.util.ME', 'cylc') + monkeypatch.setattr( + 'cylc.flow.tui.util._display_workflow_id', + lambda data: data['name'].rsplit('/', 1)[-1] + ) + monkeypatch.setattr( + 'cylc.flow.tui.overlay._get_display_id', + get_display_id, + ) + + # filter Tui so that only workflows created within our test show up + id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser())) + workflow_filter = re.escape(id_base) + r'/.*' + + @contextmanager + def _raikura(workflow_id=None, size='80,50'): + screen, html_fragment = configure_screenshot(size) + app = TuiApp(screen=screen) + with app.main( + workflow_id, + id_filter=workflow_filter, + interactive=False, + ): + yield RaikuraSession( + app, + html_fragment, + test_dir, + request.function.__name__, + ) + + return _raikura diff --git a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html new file mode 100644 index 00000000000..f5a19fd428d --- /dev/null +++ b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html @@ -0,0 +1,21 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+           ̿○ b                                                                  
+      - ̿○ 2                                                                     
+         - ̿○ A                                                                  
+              ̿○ a                                                               
+           ○ b                                                                  
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_auto_expansion.on-load.html b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html new file mode 100644 index 00000000000..df3c9f5c41b --- /dev/null +++ b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html @@ -0,0 +1,21 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+         - ̿○ A                                                                  
+              ̿○ a                                                               
+           ○ b                                                                  
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_errors.list-error.html b/tests/integration/tui/screenshots/test_errors.list-error.html new file mode 100644 index 00000000000..02448aa0267 --- /dev/null +++ b/tests/integration/tui/screenshots/test_errors.list-error.html @@ -0,0 +1,31 @@ +
┌────────────────┌────────────────────────────────────────────────┐────────────┐
+│  Error: Somethi│  Error                                         │            │
+│                │                                                │            │
+│  < Select File │  Something went wrong :(                       │         >  │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│                │                                                │            │
+│ q to close     │ q to close                                     │            │
+└────────────────└────────────────────────────────────────────────┘────────────┘
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_errors.open-error.html b/tests/integration/tui/screenshots/test_errors.open-error.html new file mode 100644 index 00000000000..142d0d88c72 --- /dev/null +++ b/tests/integration/tui/screenshots/test_errors.open-error.html @@ -0,0 +1,31 @@ +
┌──────────────────────────────────────────────────────────────────────────────┐
+│  Error: Something went wrong :(                                              │
+│                                                                              │
+│  < Select File                                                            >  │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│ q to close                                                                   │
+└──────────────────────────────────────────────────────────────────────────────┘
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_job_logs.01-job.out.html b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html new file mode 100644 index 00000000000..c1c767b98cf --- /dev/null +++ b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html @@ -0,0 +1,31 @@ +
┌──────────────────────────────────────────────────────────────────────────────┐
+│  Host: myhost                                                                │
+│  Path: mypath                                                                │
+│  < Select File                                                            >  │
+│                                                                              │
+│  job: 1/a/01                                                                 │
+│  this is a job log                                                           │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│ q to close                                                                   │
+└──────────────────────────────────────────────────────────────────────────────┘
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_job_logs.02-job.out.html b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html new file mode 100644 index 00000000000..0eb94051201 --- /dev/null +++ b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html @@ -0,0 +1,31 @@ +
┌──────────────────────────────────────────────────────────────────────────────┐
+│  Host: myhost                                                                │
+│  Path: mypath                                                                │
+│  < Select File                                                            >  │
+│                                                                              │
+│  job: 1/a/02                                                                 │
+│  this is a job log                                                           │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│ q to close                                                                   │
+└──────────────────────────────────────────────────────────────────────────────┘
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html new file mode 100644 index 00000000000..bf5e3812008 --- /dev/null +++ b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html @@ -0,0 +1,31 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+         + ̿○ A                                                                  
+         - ̿○ B                                                                  
+            - ̿○ B1                                                              
+                 ̿○ b11                                                          
+                 ̿○ b12                                                          
+            - ̿○ B2                                                              
+                 ̿○ b21                                                          
+                 ̿○ b22                                                          
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html new file mode 100644 index 00000000000..cefab5264f4 --- /dev/null +++ b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html @@ -0,0 +1,31 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+         + ̿○ A                                                                  
+         - ̿○ B                                                                  
+            - ̿○ B1                                                              
+                 ̿○ b11                                                          
+                 ̿○ b12                                                          
+            - ̿○ B2                                                              
+                 ̿○ b21                                                          
+                 ̿○ b22                                                          
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_navigation.on-load.html b/tests/integration/tui/screenshots/test_navigation.on-load.html new file mode 100644 index 00000000000..a0bd107742b --- /dev/null +++ b/tests/integration/tui/screenshots/test_navigation.on-load.html @@ -0,0 +1,31 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   + one - paused                                                               
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html new file mode 100644 index 00000000000..6b26ced563e --- /dev/null +++ b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html @@ -0,0 +1,31 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+         - ̿○ A                                                                  
+              ̿○ a1                                                              
+              ̿○ a2                                                              
+         - ̿○ B                                                                  
+            - ̿○ B1                                                              
+                 ̿○ b11                                                          
+                 ̿○ b12                                                          
+            - ̿○ B2                                                              
+                 ̿○ b21                                                          
+                 ̿○ b22                                                          
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html b/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html new file mode 100644 index 00000000000..88defab9486 --- /dev/null +++ b/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html @@ -0,0 +1,16 @@ +
Cylc Tui   work┌────┌────────────────────────────────────────────────┐          
+               │  id│  Error                                         │          
+- ~cylc        │    │                                                │          
+   + one - stop│  Ac│  Error in command cylc clean --yes one         │          
+               │  < │  mock-stderr                                   │          
+               │    │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │    │                                                │          
+               │    │                                                │          
+               │    │                                                │          
+quit: q  help: │ q t│ q to close                                     │ome End   
+filter tasks: T└────└────────────────────────────────────────────────┘          
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html new file mode 100644 index 00000000000..f28cced0714 --- /dev/null +++ b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html @@ -0,0 +1,16 @@ +
Cylc Tui   work┌────────────────────────────────────────────────┐               
+               │  id: ~cylc/one                                 │               
+- ~cylc        │                                                │               
+   + one - stop│  Action                                        │               
+               │  < (cancel)                                 >  │               
+               │                                                │               
+               │  < clean                                    >  │               
+               │  < log                                      >  │               
+               │  < play                                     >  │               
+               │  < reinstall-reload                         >  │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+quit: q  help: │ q to close                                     │↥ ↧ Home End   
+filter tasks: T└────────────────────────────────────────────────┘               
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html new file mode 100644 index 00000000000..c2355597f78 --- /dev/null +++ b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html @@ -0,0 +1,16 @@ +
Cylc Tui   work┌────────────────────────────────────────────────┐               
+               │  id: ~cylc/root                                │               
+- ~cylc        │                                                │               
+   + one - stop│  Action                                        │               
+               │  < (cancel)                                 >  │               
+               │                                                │               
+               │  < stop-all                                 >  │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+quit: q  help: │ q to close                                     │↥ ↧ Home End   
+filter tasks: T└────────────────────────────────────────────────┘               
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html b/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html new file mode 100644 index 00000000000..895856c6ea2 --- /dev/null +++ b/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html @@ -0,0 +1,16 @@ +
Cylc Tui   work┌────┌────────────────────────────────────────────────┐          
+               │  id│  Error                                         │          
+- ~cylc        │    │                                                │          
+   - one - paus│  Ac│  Error connecting to workflow: mock error      │          
+      - ̿○ 1    │  < │                                                │          
+           ̿○ on│    │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │    │                                                │          
+quit: q  help: │ q t│ q to close                                     │ome End   
+filter tasks: T└────└────────────────────────────────────────────────┘          
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html b/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html new file mode 100644 index 00000000000..6f9954926ef --- /dev/null +++ b/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html @@ -0,0 +1,16 @@ +
Cylc Tui   work┌────┌────────────────────────────────────────────────┐          
+               │  id│  Error                                         │          
+- ~cylc        │    │                                                │          
+   - one - paus│  Ac│  Cannot peform command hold on a stopped       │          
+      - ̿○ 1    │  < │  workflow                                      │          
+           ̿○ on│    │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │    │                                                │          
+quit: q  help: │ q t│ q to close                                     │ome End   
+filter tasks: T└────└────────────────────────────────────────────────┘          
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_online_mutation.command-failed.html b/tests/integration/tui/screenshots/test_online_mutation.command-failed.html new file mode 100644 index 00000000000..fae4a429cc6 --- /dev/null +++ b/tests/integration/tui/screenshots/test_online_mutation.command-failed.html @@ -0,0 +1,16 @@ +
Cylc Tui   work┌────┌────────────────────────────────────────────────┐          
+               │  id│  Error                                         │          
+- ~cylc        │    │                                                │          
+   - one - paus│  Ac│  Cannot peform command hold on a stopped       │          
+      - ̿○ 1    │  < │  workflow                                      │          
+           ̿○ on│    │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │    │                                                │          
+               │    │                                                │          
+quit: q  help: │ q t│ q to close                                     │ome End   
+filter tasks: T└────└────────────────────────────────────────────────┘          
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html new file mode 100644 index 00000000000..34be2ffa0ce --- /dev/null +++ b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html @@ -0,0 +1,16 @@ +
Cylc Tui   work┌────────────────────────────────────────────────┐               
+               │  id: 1/one                                     │               
+- ~cylc        │                                                │               
+   - one - paus│  Action                                        │               
+      - ̿○ 1    │  < (cancel)                                 >  │               
+           ̿○ on│                                                │               
+               │  < hold                                     >  │               
+               │  < kill                                     >  │               
+               │  < log                                      >  │               
+               │  < poll                                     >  │               
+               │  < release                                  >  │               
+               │  < show                                     >  │               
+               │                                                │               
+quit: q  help: │ q to close                                     │↥ ↧ Home End   
+filter tasks: T└────────────────────────────────────────────────┘               
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_online_mutation.task-selected.html b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html new file mode 100644 index 00000000000..7d94d5e43dd --- /dev/null +++ b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html @@ -0,0 +1,16 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+           ̿○ one                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html new file mode 100644 index 00000000000..74c02508239 --- /dev/null +++ b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html @@ -0,0 +1,21 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+           ̿○ one                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html new file mode 100644 index 00000000000..09c3bbd7fb0 --- /dev/null +++ b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html @@ -0,0 +1,21 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - stopped                                                              
+        Workflow is not running                                                 
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html new file mode 100644 index 00000000000..74c02508239 --- /dev/null +++ b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html @@ -0,0 +1,21 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+           ̿○ one                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html new file mode 100644 index 00000000000..f88e1b0124d --- /dev/null +++ b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html @@ -0,0 +1,31 @@ +
┌──────────────────────────────────────────────────────────────────────────────┐
+│  Host: myhost                                                                │
+│  Path: mypath                                                                │
+│  < Select File                                                            >  │
+│                                                                              │
+│  this is the                                                                 │
+│  scheduler log file                                                          │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                     ┌──────────────────────────────────────┐                 │
+│                     │  Select File                         │                 │
+│                     │                                      │                 │
+│                     │  < config/01-start-01.cylc        >  │                 │
+│                     │  < config/flow-processed.cylc     >  │                 │
+│                     │  < scheduler/01-start-01.log      >  │                 │
+│                     │                                      │                 │
+│                     │ q to close                           │                 │
+│                     └──────────────────────────────────────┘                 │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│ q to close                                                                   │
+└──────────────────────────────────────────────────────────────────────────────┘
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html new file mode 100644 index 00000000000..68dbcc10f9c --- /dev/null +++ b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html @@ -0,0 +1,31 @@ +
┌──────────────────────────────────────────────────────────────────────────────┐
+│  Host: myhost                                                                │
+│  Path: mypath                                                                │
+│  < Select File                                                            >  │
+│                                                                              │
+│  this is the                                                                 │
+│  scheduler log file                                                          │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│ q to close                                                                   │
+└──────────────────────────────────────────────────────────────────────────────┘
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html new file mode 100644 index 00000000000..e3fcdfbab22 --- /dev/null +++ b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html @@ -0,0 +1,31 @@ +
┌──────────────────────────────────────────────────────────────────────────────┐
+│  Host: myhost                                                                │
+│  Path: mypath                                                                │
+│  < Select File                                                            >  │
+│                                                                              │
+│  [scheduling]                                                                │
+│      [[graph]]                                                               │
+│          R1 = a                                                              │
+│  [runtime]                                                                   │
+│      [[a]]                                                                   │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│ q to close                                                                   │
+└──────────────────────────────────────────────────────────────────────────────┘
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_show.fail.html b/tests/integration/tui/screenshots/test_show.fail.html new file mode 100644 index 00000000000..f788e5b3a55 --- /dev/null +++ b/tests/integration/tui/screenshots/test_show.fail.html @@ -0,0 +1,41 @@ +
Cylc Tui   workflows┌────────────────────────────────────────────────┐          
+                    │  Error                                         │          
+- ~cylc             │                                                │          
+   - one - paused   │  :(                                            │          
+      - ̿○ 1         │                                                │          
+           ̿○ foo    │                                                │          
+                    │                                                │          
+                    │                                                │          
+                    │                                                │          
+                    │                                                │          
+               ┌────│                                                │          
+               │  id│                                                │          
+               │    │                                                │          
+               │  Ac│                                                │          
+               │  < │                                                │          
+               │    │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │  < │                                                │          
+               │    │                                                │          
+               │    │                                                │          
+               │    │                                                │          
+               │    │                                                │          
+               │    │                                                │          
+               │ q t│                                                │          
+               └────│                                                │          
+                    │                                                │          
+                    │                                                │          
+                    │                                                │          
+                    │                                                │          
+                    │                                                │          
+                    │                                                │          
+                    │                                                │          
+                    │                                                │          
+quit: q  help: h  co│ q to close                                     │ome End   
+filter tasks: T f s └────────────────────────────────────────────────┘          
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_show.success.html b/tests/integration/tui/screenshots/test_show.success.html new file mode 100644 index 00000000000..afdcd1a73b4 --- /dev/null +++ b/tests/integration/tui/screenshots/test_show.success.html @@ -0,0 +1,41 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+           ̿○ foo                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+               ┌────────────────────────────────────────────────┐               
+               │  title: Foo                                    │               
+               │  description: The first metasyntactic          │               
+               │  variable.                                     │               
+               │  URL: (not given)                              │               
+               │  state: waiting                                │               
+               │  prerequisites: (None)                         │               
+               │  outputs: ('-': not completed)                 │               
+               │    - 1/foo expired                             │               
+               │    - 1/foo submitted                           │               
+               │    - 1/foo submit-failed                       │               
+               │    - 1/foo started                             │               
+               │    - 1/foo succeeded                           │               
+               │    - 1/foo failed                              │               
+               │                                                │               
+               │                                                │               
+               │ q to close                                     │               
+               └────────────────────────────────────────────────┘               
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html new file mode 100644 index 00000000000..019184ec897 --- /dev/null +++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html @@ -0,0 +1,16 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+           ̿○ one                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html new file mode 100644 index 00000000000..8fa0f4329a1 --- /dev/null +++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html @@ -0,0 +1,16 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   + one - paused                                                               
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html new file mode 100644 index 00000000000..4814892df7a --- /dev/null +++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html @@ -0,0 +1,31 @@ +
┌──────────────────────────────────────────────────────────────────────────────┐
+│  Host: myhost                                                                │
+│  Path: mypath                                                                │
+│  < Select File                                                            >  │
+│                                                                              │
+│  job: 1/a/02                                                                 │
+│  this is a job error                                                         │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│ q to close                                                                   │
+└──────────────────────────────────────────────────────────────────────────────┘
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html new file mode 100644 index 00000000000..0eb94051201 --- /dev/null +++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html @@ -0,0 +1,31 @@ +
┌──────────────────────────────────────────────────────────────────────────────┐
+│  Host: myhost                                                                │
+│  Path: mypath                                                                │
+│  < Select File                                                            >  │
+│                                                                              │
+│  job: 1/a/02                                                                 │
+│  this is a job log                                                           │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│                                                                              │
+│ q to close                                                                   │
+└──────────────────────────────────────────────────────────────────────────────┘
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-raikura-enter.html b/tests/integration/tui/screenshots/test_tui_basics.test-raikura-enter.html new file mode 100644 index 00000000000..d54d9538d26 --- /dev/null +++ b/tests/integration/tui/screenshots/test_tui_basics.test-raikura-enter.html @@ -0,0 +1,41 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+               ┌────────────────────────────────────────────────┐               
+               │  id: ~cylc/root                                │               
+               │                                                │               
+               │  Action                                        │               
+               │  < (cancel)                                 >  │               
+               │                                                │               
+               │  < stop-all                                 >  │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │                                                │               
+               │ q to close                                     │               
+               └────────────────────────────────────────────────┘               
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-raikura-help.html b/tests/integration/tui/screenshots/test_tui_basics.test-raikura-help.html new file mode 100644 index 00000000000..1795c586d9a --- /dev/null +++ b/tests/integration/tui/screenshots/test_tui_basics.test-raikura-help.html @@ -0,0 +1,41 @@ +
Cylc Tui  ┌──────────────────────────────────────────────────────────┐          
+          │                                                          │          
+- ~cylc   │                     _        _         _                 │          
+          │                    | |      | |       (_)                │          
+          │           ___ _   _| | ___  | |_ _   _ _                 │          
+          │          / __| | | | |/ __| | __| | | | |                │          
+          │         | (__| |_| | | (__  | |_| |_| | |                │          
+          │          \___|\__, |_|\___|  \__|\__,_|_|                │          
+          │                __/ |                                     │          
+          │               |___/                                      │          
+          │                                                          │          
+          │            ( scroll using arrow keys )                   │          
+          │                                                          │          
+          │                                                          │          
+          │                                                          │          
+          │                             _,@@@@@@.                    │          
+          │                           <=@@@, `@@@@@.                 │          
+          │                              `-@@@@@@@@@@@'              │          
+          │                                 :@@@@@@@@@@.             │          
+          │                                (.@@@@@@@@@@@             │          
+          │                               ( '@@@@@@@@@@@@.           │          
+          │                              ;.@@@@@@@@@@@@@@@           │          
+          │                            '@@@@@@@@@@@@@@@@@@,          │          
+          │                          ,@@@@@@@@@@@@@@@@@@@@'          │          
+          │                        :.@@@@@@@@@@@@@@@@@@@@@.          │          
+          │                      .@@@@@@@@@@@@@@@@@@@@@@@@.          │          
+          │                    '@@@@@@@@@@@@@@@@@@@@@@@@@.           │          
+          │                  ;@@@@@@@@@@@@@@@@@@@@@@@@@@@            │          
+          │                 .@@@@@@@@@@@@@@@@@@@@@@@@@@.             │          
+          │                .@@@@@@@@@@@@@@@@@@@@@@@@@@,              │          
+          │               .@@@@@@@@@@@@@@@@@@@@@@@@@'                │          
+          │              .@@@@@@@@@@@@@@@@@@@@@@@@'     ,            │          
+          │            :@@@@@@@@@@@@@@@@@@@@@..''';,,,;::-           │          
+          │           '@@@@@@@@@@@@@@@@@@@.        `.   `            │          
+          │          .@@@@@@.: ,.@@@@@@@.            `               │          
+          │        :@@@@@@@,         ;.@,                            │          
+          │       '@@@@@@.              `@'                          │          
+          │                                                          │          
+quit: q  h│ q to close                                               │ome End   
+filter tas└──────────────────────────────────────────────────────────┘          
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-raikura.html b/tests/integration/tui/screenshots/test_tui_basics.test-raikura.html new file mode 100644 index 00000000000..7f80031804b --- /dev/null +++ b/tests/integration/tui/screenshots/test_tui_basics.test-raikura.html @@ -0,0 +1,41 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-active.html b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html new file mode 100644 index 00000000000..282f76735ed --- /dev/null +++ b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html @@ -0,0 +1,16 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   + one - stopping                                                             
+   + two - paused                                                               
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html new file mode 100644 index 00000000000..8c26ce6ccc9 --- /dev/null +++ b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html @@ -0,0 +1,16 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   + tre - stopped                                                              
+   + two - paused                                                               
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html new file mode 100644 index 00000000000..8c26ce6ccc9 --- /dev/null +++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html @@ -0,0 +1,16 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   + tre - stopped                                                              
+   + two - paused                                                               
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html new file mode 100644 index 00000000000..1ff602df101 --- /dev/null +++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html @@ -0,0 +1,16 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   + tre - stopped                                                              
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html new file mode 100644 index 00000000000..0651eedec30 --- /dev/null +++ b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html @@ -0,0 +1,16 @@ +
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   + one - stopping                                                             
+   + tre - stopped                                                              
+   + two - paused                                                               
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+
\ No newline at end of file diff --git a/tests/integration/tui/test_app.py b/tests/integration/tui/test_app.py new file mode 100644 index 00000000000..d7dc26457b3 --- /dev/null +++ b/tests/integration/tui/test_app.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import pytest +import urwid + +from cylc.flow.cycling.integer import IntegerPoint +from cylc.flow.task_state import ( +# TASK_STATUS_RUNNING, + TASK_STATUS_SUCCEEDED, +# TASK_STATUS_FAILED, +# TASK_STATUS_WAITING, +) +from cylc.flow.workflow_status import StopMode + + +def set_task_state(schd, task_states): + """Force tasks into the desired states. + + Task states should be of the format (cycle, task, state, is_held). + """ + for cycle, task, state, is_held in task_states: + itask = schd.pool.get_task(cycle, task) + if not itask: + itask = schd.pool.spawn_task(task, cycle, {1}) + itask.state_reset(state, is_held=is_held) + schd.data_store_mgr.delta_task_state(itask) + schd.data_store_mgr.increment_graph_window( + itask.tokens, + cycle, + {1}, + ) + + +async def test_tui_basics(raikura): + """Test basic Tui interaction with no workflows.""" + with raikura(size='80,40') as rk: + # the app should open + rk.compare_screenshot('test-raikura', 'the app should have loaded') + + # "h" should bring up the onscreen help + rk.user_input('h') + rk.compare_screenshot( + 'test-raikura-help', + 'the help screen should be visible' + ) + + # "q" should close the popup + rk.user_input('q') + rk.compare_screenshot( + 'test-raikura', + 'the help screen should have closed', + ) + + # "enter" should bring up the context menu + rk.user_input('enter') + rk.compare_screenshot( + 'test-raikura-enter', + 'the context menu should have opened', + ) + + # "enter" again should close it via the "cancel" button + rk.user_input('enter') + rk.compare_screenshot( + 'test-raikura', + 'the context menu should have closed', + ) + + # "ctrl d" should exit Tui + with pytest.raises(urwid.ExitMainLoop): + rk.user_input('ctrl d') + + # "q" should exit Tui + with pytest.raises(urwid.ExitMainLoop): + rk.user_input('q') + + +async def test_subscribe_unsubscribe(one_conf, flow, scheduler, start, raikura): + """Test a simple workflow with one task.""" + id_ = flow(one_conf, name='one') + schd = scheduler(id_) + async with start(schd): + await schd.update_data_structure() + with raikura(size='80,15') as rk: + rk.compare_screenshot( + 'unsubscribed', + 'the workflow should be collapsed' + ' (no subscription no state totals)', + ) + + # expand the workflow to subscribe to it + rk.user_input('down', 'right') + rk.wait_until_loaded() + rk.compare_screenshot( + 'subscribed', + 'the workflow should be expanded', + ) + + # collapse the workflow to unsubscribe from it + rk.user_input('left', 'up') + rk.force_update() + rk.compare_screenshot( + 'unsubscribed', + 'the workflow should be collapsed' + ' (no subscription no state totals)', + ) + + +async def test_workflow_states(one_conf, flow, scheduler, start, raikura): + """Test viewing multiple workflows in different states.""" + # one => stopping + id_1 = flow(one_conf, name='one') + schd_1 = scheduler(id_1) + # two => paused + id_2 = flow(one_conf, name='two') + schd_2 = scheduler(id_2) + # tre => stopped + flow(one_conf, name='tre') + + async with start(schd_1): + schd_1.stop_mode = StopMode.AUTO # make it look like we're stopping + await schd_1.update_data_structure() + + async with start(schd_2): + await schd_2.update_data_structure() + with raikura(size='80,15') as rk: + rk.compare_screenshot( + 'unfiltered', + 'All workflows should be visible (one, two, tree)', + ) + + # filter for active workflows (i.e. paused, running, stopping) + rk.user_input('p') + rk.compare_screenshot( + 'filter-active', + 'Only active workflows should be visible (one, two)' + ) + + # invert the filter so we are filtering for stopped workflows + rk.user_input('W', 'enter', 'q') + rk.compare_screenshot( + 'filter-stopped', + 'Only stopped workflow should be visible (tre)' + ) + + # filter in paused workflows + rk.user_input('W', 'down', 'enter', 'q') + rk.force_update() + rk.compare_screenshot( + 'filter-stopped-or-paused', + 'Only stopped or paused workflows should be visible' + ' (two, tre)', + ) + + # reset the state filters + rk.user_input('W', 'down', 'down', 'enter', 'down', 'enter') + + # scroll to the id filter text box + rk.user_input('down', 'down', 'down', 'down') + + # scroll to the end of the ID + rk.user_input(*['right'] * ( + len(schd_1.tokens['workflow'].rsplit('/', 1)[0]) + 1) + ) + + # type the letter "t" + # (this should filter for workflows starting with "t") + rk.user_input('t') + rk.force_update() # this is required for the tests + rk.user_input('page up', 'q') # close the dialogue + + rk.compare_screenshot( + 'filter-starts-with-t', + 'Only workflows starting with the letter "t" should be' + ' visible (two, tre)', + ) + + +# TODO: Task state filtering is currently broken +# see: https://github.com/cylc/cylc-flow/issues/5716 +# +# async def test_task_states(flow, scheduler, start, raikura): +# id_ = flow({ +# 'scheduler': { +# 'allow implicit tasks': 'true', +# }, +# 'scheduling': { +# 'initial cycle point': '1', +# 'cycling mode': 'integer', +# 'runahead limit': 'P1', +# 'graph': { +# 'P1': ''' +# a => b => c +# b[-P1] => b +# ''' +# } +# } +# }, name='test_task_states') +# schd = scheduler(id_) +# async with start(schd): +# set_task_state( +# schd, +# [ +# (IntegerPoint('1'), 'a', TASK_STATUS_SUCCEEDED, False), +# # (IntegerPoint('1'), 'b', TASK_STATUS_FAILED, False), +# (IntegerPoint('1'), 'c', TASK_STATUS_RUNNING, False), +# # (IntegerPoint('2'), 'a', TASK_STATUS_RUNNING, False), +# (IntegerPoint('2'), 'b', TASK_STATUS_WAITING, True), +# ] +# ) +# await schd.update_data_structure() +# +# with raikura(schd.tokens.id, size='80,20') as rk: +# rk.compare_screenshot('unfiltered') +# +# # filter out waiting tasks +# rk.user_input('T', 'down', 'enter', 'q') +# rk.compare_screenshot('filter-not-waiting') + + +async def test_navigation(flow, scheduler, start, raikura): + """Test navigating with the arrow keys.""" + id_ = flow({ + 'scheduling': { + 'graph': { + 'R1': 'A & B1 & B2', + } + }, + 'runtime': { + 'A': {}, + 'B': {}, + 'B1': {'inherit': 'B'}, + 'B2': {'inherit': 'B'}, + 'a1': {'inherit': 'A'}, + 'a2': {'inherit': 'A'}, + 'b11': {'inherit': 'B1'}, + 'b12': {'inherit': 'B1'}, + 'b21': {'inherit': 'B2'}, + 'b22': {'inherit': 'B2'}, + } + }, name='one') + schd = scheduler(id_) + async with start(schd): + await schd.update_data_structure() + + with raikura(size='80,30') as rk: + rk.compare_screenshot( + 'on-load', + 'the workflow should be collapsed when Tui is loaded', + ) + + # pressing "right" should connect to the workflow + # and expand it once the data arrives + rk.user_input('down', 'right') + rk.wait_until_loaded(schd.tokens.id) + rk.compare_screenshot( + 'workflow-expanded', + 'the workflow should be expanded', + ) + + # pressing "left" should collapse the node + rk.user_input('down', 'down', 'left') + rk.compare_screenshot( + 'family-A-collapsed', + 'the family "1/A" should be collapsed', + ) + + # the "page up" and "page down" buttons should navigate to the top + # and bottom of the screen + rk.user_input('page down') + rk.compare_screenshot( + 'cursor-at-bottom-of-screen', + 'the cursor should be at the bottom of the screen', + ) + + +async def test_auto_expansion(flow, scheduler, start, raikura): + """It should automatically expand cycles and top-level families. + + When a workflow is expanded, Tui should auto expand cycles and top-level + families. Any new cycles and top-level families should be auto-expanded + when added. + """ + id_ = flow({ + 'scheduling': { + 'runahead limit': 'P1', + 'initial cycle point': '1', + 'cycling mode': 'integer', + 'graph': { + 'P1': 'b[-P1] => a => b' + }, + }, + 'runtime': { + 'A': {}, + 'a': {'inherit': 'A'}, + 'b': {}, + }, + }, name='one') + schd = scheduler(id_) + with raikura(size='80,20') as rk: + async with start(schd): + await schd.update_data_structure() + + # open the workflow + rk.force_update() + rk.user_input('down', 'right') + rk.wait_until_loaded(schd.tokens.id) + + rk.compare_screenshot( + 'on-load', + 'cycle "1" and top-level family "1/A" should be expanded', + ) + + for task in ('a', 'b'): + itask = schd.pool.get_task(IntegerPoint('1'), task) + itask.state_reset(TASK_STATUS_SUCCEEDED) + schd.pool.spawn_on_output(itask, TASK_STATUS_SUCCEEDED) + await schd.update_data_structure() + + rk.compare_screenshot( + 'later-time', + 'cycle "2" and top-level family "2/A" should be expanded', + ) + + +async def test_restart_reconnect(one_conf, flow, scheduler, start, raikura): + """It should handle workflow shutdown and restart. + + The Cylc client can raise exceptions e.g. WorkflowStopped. Any text written + to stdout/err will mess with Tui. The purpose of this test is to ensure Tui + can handle shutdown / restart without any errors occuring and any spurious + text appearing on the screen. + """ + with raikura(size='80,20') as rk: + schd = scheduler(flow(one_conf, name='one')) + + # 1- start the workflow + async with start(schd): + await schd.update_data_structure() + rk.force_update() + rk.user_input('down', 'right') + rk.wait_until_loaded(schd.tokens.id) + rk.compare_screenshot( + '1-workflow-running', + 'the workflow should appear in tui and be expanded on load', + ) + + # 2 - stop the worlflow + rk.compare_screenshot( + '2-workflow-stopped', + 'the stopped workflow should be collapsed with a message saying' + ' workflow stopped', + ) + + # 3- restart the workflow + schd = scheduler(flow(one_conf, name='one')) + async with start(schd): + await schd.update_data_structure() + rk.wait_until_loaded(schd.tokens.id) + # rk.user_input('down', 'right') + rk.compare_screenshot( + '3-workflow-restarted', + 'the restarted workflow should be expanded', + ) diff --git a/tests/integration/tui/test_logs.py b/tests/integration/tui/test_logs.py new file mode 100644 index 00000000000..7b734030bb7 --- /dev/null +++ b/tests/integration/tui/test_logs.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import asyncio +from pathlib import Path +from typing import TYPE_CHECKING + +from cylc.flow.cycling.integer import IntegerPoint +from cylc.flow.exceptions import ClientError +from cylc.flow.task_job_logs import get_task_job_log +from cylc.flow.task_state import ( + TASK_STATUS_FAILED, + TASK_STATUS_SUCCEEDED, +) +from cylc.flow.tui.data import _get_log + +import pytest + +if TYPE_CHECKING: + from cylc.flow.id import Tokens + + +def get_job_log(tokens: 'Tokens', suffix: str) -> Path: + """Return the path to a job log file. + + Args: + tokens: Job tokens. + suffix: Filename. + + """ + return Path(get_task_job_log( + tokens['workflow'], + tokens['cycle'], + tokens['task'], + tokens['job'], + suffix=suffix, + )) + + +@pytest.fixture(scope='module') +def standarise_host_and_path(mod_monkeypatch): + """Replace variable content in the log view. + + The log view displays the "Host" and "Path" of the log file. These will + differer from user to user, so we mock away the difference to produce + stable results. + """ + def _parse_log_header(contents): + _header, text = contents.split('\n', 1) + return 'myhost', 'mypath', text + + mod_monkeypatch.setattr( + 'cylc.flow.tui.data._parse_log_header', + _parse_log_header, + ) + + +@pytest.fixture +def wait_log_loaded(monkeypatch): + """Wait for Tui to successfully open a log file.""" + # previous log open count + before = 0 + # live log open count + count = 0 + + # wrap the Tui "_get_log" method to count the number of times it has + # returned + def __get_log(*args, **kwargs): + nonlocal count + try: + ret = _get_log(*args, **kwargs) + except ClientError as exc: + count += 1 + raise exc + count += 1 + return ret + monkeypatch.setattr( + 'cylc.flow.tui.data._get_log', + __get_log, + ) + + async def _wait_log_loaded(tries: int = 25, delay: float = 0.1): + """Wait for the log file to be loaded. + + Args: + tries: The number of (re)tries to attempt before failing. + delay: The delay between retries. + + """ + nonlocal before, count + for _try in range(tries): + if count > before: + await asyncio.sleep(0) + before += 1 + return + await asyncio.sleep(delay) + raise Exception(f'Log file was not loaded within {delay * tries}s') + + return _wait_log_loaded + + +@pytest.fixture(scope='module') +async def workflow(mod_flow, mod_scheduler, mod_start, standarise_host_and_path): + """Test fixture providing a workflow with some log files to poke at.""" + id_ = mod_flow({ + 'scheduling': { + 'graph': { + 'R1': 'a', + } + }, + 'runtime': { + 'a': {}, + } + }, name='one') + schd = mod_scheduler(id_) + async with mod_start(schd): + # create some log files for tests to inspect + + # create a scheduler log + # (note the scheduler log doesn't get created in integration tests) + scheduler_log = Path(schd.workflow_log_dir, '01-start-01.log') + with open(scheduler_log, 'w+') as logfile: + logfile.write('this is the\nscheduler log file') + + # task 1/a + itask = schd.pool.get_task(IntegerPoint('1'), 'a') + itask.submit_num = 2 + + # mark 1/a/01 as failed + job_1 = schd.tokens.duplicate(cycle='1', task='a', job='01') + schd.data_store_mgr.insert_job( + 'a', + IntegerPoint('1'), + TASK_STATUS_SUCCEEDED, + {'submit_num': 1, 'platform': {'name': 'x'}} + ) + schd.data_store_mgr.delta_job_state(job_1, TASK_STATUS_FAILED) + + # mark 1/a/02 as succeeded + job_2 = schd.tokens.duplicate(cycle='1', task='a', job='02') + schd.data_store_mgr.insert_job( + 'a', + IntegerPoint('1'), + TASK_STATUS_SUCCEEDED, + {'submit_num': 2, 'platform': {'name': 'x'}} + ) + schd.data_store_mgr.delta_job_state(job_1, TASK_STATUS_SUCCEEDED) + schd.data_store_mgr.delta_task_state(itask) + + # mark 1/a as succeeded + itask.state_reset(TASK_STATUS_SUCCEEDED) + schd.data_store_mgr.delta_task_state(itask) + + # 1/a/01 - job.out + job_1_out = get_job_log(job_1, 'job.out') + job_1_out.parent.mkdir(parents=True) + with open(job_1_out, 'w+') as log: + log.write(f'job: {job_1.relative_id}\nthis is a job log\n') + + # 1/a/02 - job.out + job_2_out = get_job_log(job_2, 'job.out') + job_2_out.parent.mkdir(parents=True) + with open(job_2_out, 'w+') as log: + log.write(f'job: {job_2.relative_id}\nthis is a job log\n') + + # 1/a/02 - job.err + job_2_err = get_job_log(job_2, 'job.err') + with open(job_2_err, 'w+') as log: + log.write(f'job: {job_2.relative_id}\nthis is a job error\n') + + # 1/a/NN -> 1/a/02 + (job_2_out.parent.parent / 'NN').symlink_to( + (job_2_out.parent.parent / '02'), + target_is_directory=True, + ) + + # populate the data store + await schd.update_data_structure() + + yield schd + + +async def test_scheduler_logs( + workflow, + mod_raikura, + wait_log_loaded, +): + """Test viewing the scheduler log files.""" + with mod_raikura(size='80,30') as rk: + # open the workflow in Tui + rk.user_input('down', 'right') + rk.wait_until_loaded(workflow.tokens.id) + + # open the log view for the workflow + rk.user_input('enter') + rk.user_input('down', 'down', 'enter') + + # wait for the default log file to load + await wait_log_loaded() + rk.compare_screenshot( + 'scheduler-log-file', + 'the scheduler log file should be open', + ) + + # open the list of log files + rk.user_input('enter') + rk.compare_screenshot( + 'log-file-selection', + 'the list of available log files should be displayed' + ) + + # select the processed workflow configuration file + rk.user_input('down', 'enter') + + # wait for the file to load + await wait_log_loaded() + rk.compare_screenshot( + 'workflow-configuration-file', + 'the workflow configuration file should be open' + ) + + +async def test_task_logs( + workflow, + mod_raikura, + wait_log_loaded, +): + """Test viewing task log files. + + I.E. Test viewing job log files by opening the log view on a task. + """ + with mod_raikura(size='80,30') as rk: + # open the workflow in Tui + rk.user_input('down', 'right') + rk.wait_until_loaded(workflow.tokens.id) + + # open the context menu for the task 1/a + rk.user_input('down', 'down', 'enter') + + # open the log view for the task 1/a + rk.user_input('down', 'down', 'down', 'enter') + + # wait for the default log file to load + await wait_log_loaded() + rk.compare_screenshot( + 'latest-job.out', + 'the job.out file for the second job should be open', + ) + + rk.user_input('enter') + rk.user_input('enter') + + # wait for the job.err file to load + await wait_log_loaded() + rk.compare_screenshot( + 'latest-job.err', + 'the job.out file for the second job should be open', + ) + + +async def test_job_logs( + workflow, + mod_raikura, + wait_log_loaded, +): + """Test viewing the job log files. + + I.E. Test viewing job log files by opening the log view on a job. + """ + with mod_raikura(size='80,30') as rk: + # open the workflow in Tui + rk.user_input('down', 'right') + rk.wait_until_loaded(workflow.tokens.id) + + # open the context menu for the job 1/a/02 + rk.user_input('down', 'down', 'right', 'down', 'enter') + + # open the log view for the job 1/a/02 + rk.user_input('down', 'down', 'down', 'enter') + + # wait for the default log file to load + await wait_log_loaded() + rk.compare_screenshot( + '02-job.out', + 'the job.out file for the *second* job should be open', + ) + + # close log view + rk.user_input('q') + + # open the log view for the job 1/a/01 + rk.user_input('down', 'enter') + rk.user_input('down', 'down', 'down', 'enter') + + # wait for the default log file to load + await wait_log_loaded() + rk.compare_screenshot( + '01-job.out', + 'the job.out file for the *first* job should be open', + ) + + +async def test_errors( + workflow, + mod_raikura, + wait_log_loaded, + monkeypatch, +): + """Test error handing of cat-log commands.""" + # make it look like cat-log commands are failing + def cli_cmd_fail(*args, **kwargs): + raise ClientError('Something went wrong :(') + + monkeypatch.setattr( + 'cylc.flow.tui.data.cli_cmd', + cli_cmd_fail, + ) + + with mod_raikura(size='80,30') as rk: + # open the log view on scheduler + rk.user_input('down', 'enter', 'down', 'down', 'enter') + + # it will fail to open + await wait_log_loaded() + rk.compare_screenshot( + 'open-error', + 'the error message should be displayed in the log view header', + ) + + # open the file selector + rk.user_input('enter') + + # it will fail to list avialable log files + rk.compare_screenshot( + 'list-error', + 'the error message should be displayed in a pop up', + ) diff --git a/tests/integration/tui/test_mutations.py b/tests/integration/tui/test_mutations.py new file mode 100644 index 00000000000..659c74ac1d0 --- /dev/null +++ b/tests/integration/tui/test_mutations.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import asyncio + +import pytest + +from cylc.flow.exceptions import ClientError + + +async def gen_commands(schd): + """Yield commands from the scheduler's command queue.""" + while True: + await asyncio.sleep(0.1) + if not schd.command_queue.empty(): + yield schd.command_queue.get() + + +async def test_online_mutation( + one_conf, + flow, + scheduler, + start, + raikura, + monkeypatch, +): + """Test a simple workflow with one task.""" + id_ = flow(one_conf, name='one') + schd = scheduler(id_) + with raikura(size='80,15') as rk: + async with start(schd): + await schd.update_data_structure() + assert schd.command_queue.empty() + + # open the workflow + rk.force_update() + rk.user_input('down', 'right') + rk.wait_until_loaded(schd.tokens.id) + + # focus on a task + rk.user_input('down', 'right', 'down', 'right') + rk.compare_screenshot( + # take a screenshot to ensure we have focused on the task + # successfully + 'task-selected', + 'the cursor should be on the task 1/foo', + ) + + # focus on the hold mutation for a task + rk.user_input('enter', 'down') + rk.compare_screenshot( + # take a screenshot to ensure we have focused on the mutation + # successfully + 'hold-mutation-selected', + 'the cursor should be on the "hold" mutation', + ) + + # run the hold mutation + rk.user_input('enter') + + # the mutation should be in the scheduler's command_queue + command = None + async for command in gen_commands(schd): + break + assert command == ('hold', (['1/one'],), {}) + + # close the dialogue and re-run the hold mutation + rk.user_input('q', 'q', 'enter') + rk.compare_screenshot( + 'command-failed-workflow-stopped', + 'an error should be visible explaining that the operation' + ' cannot be performed on a stopped workflow', + # NOTE: don't update so Tui still thinks the workflow is running + force_update=False, + ) + + # force mutations to raise ClientError + def _get_client(*args, **kwargs): + raise ClientError('mock error') + monkeypatch.setattr( + 'cylc.flow.tui.data.get_client', + _get_client, + ) + + # close the dialogue and re-run the hold mutation + rk.user_input('q', 'q', 'enter') + rk.compare_screenshot( + 'command-failed-client-error', + 'an error should be visible explaining that the operation' + ' failed due to a client error', + # NOTE: don't update so Tui still thinks the workflow is running + force_update=False, + ) + + +@pytest.fixture +def standardise_cli_cmds(monkeypatch): + """This remove the variable bit of the workflow ID from CLI commands. + + The workflow ID changes from run to run. In order to make screenshots + stable, this + """ + from cylc.flow.tui.data import extract_context + def _extract_context(selection): + context = extract_context(selection) + if 'workflow' in context: + context['workflow'] = [ + workflow.rsplit('/', 1)[-1] + for workflow in context.get('workflow', []) + ] + return context + monkeypatch.setattr( + 'cylc.flow.tui.data.extract_context', + _extract_context, + ) + +@pytest.fixture +def capture_commands(monkeypatch): + ret = [] + returncode = [0] + + class _Popen: + def __init__(self, *args, **kwargs): + nonlocal ret + ret.append(args) + + def communicate(self): + return 'mock-stdout', 'mock-stderr' + + @property + def returncode(self): + nonlocal returncode + return returncode[0] + + monkeypatch.setattr( + 'cylc.flow.tui.data.Popen', + _Popen, + ) + + return ret, returncode + + +async def test_offline_mutation( + one_conf, + flow, + raikura, + capture_commands, + standardise_cli_cmds, +): + id_ = flow(one_conf, name='one') + commands, returncode = capture_commands + + with raikura(size='80,15') as rk: + # run the stop-all mutation + rk.wait_until_loaded('root') + rk.user_input('enter', 'down') + rk.compare_screenshot( + # take a screenshot to ensure we have focused on the task + # successfully + 'stop-all-mutation-selected', + 'the stop-all mutation should be selected', + ) + rk.user_input('enter') + + # the command "cylc stop '*'" should have been run + assert commands == [(['cylc', 'stop', '*'],)] + commands.clear() + + # run the clean command on the workflow + rk.user_input('down', 'enter', 'down') + rk.compare_screenshot( + # take a screenshot to ensure we have focused on the mutation + # successfully + 'clean-mutation-selected', + 'the clean mutation should be selected', + ) + rk.user_input('enter') + + # the command "cylc clean " should have been run + assert commands == [(['cylc', 'clean', '--yes', 'one'],)] + commands.clear() + + # make commands fail + returncode[:] = [1] + rk.user_input('enter', 'down') + rk.compare_screenshot( + # take a screenshot to ensure we have focused on the mutation + # successfully + 'clean-mutation-selected', + 'the clean mutation should be selected', + ) + rk.user_input('enter') + + assert commands == [(['cylc', 'clean', '--yes', 'one'],)] + + rk.compare_screenshot( + # take a screenshot to ensure we have focused on the mutation + # successfully + 'clean-command-error', + 'there should be a box displaying the error containing the stderr' + ' returned by the command', + ) diff --git a/tests/integration/tui/test_show.py b/tests/integration/tui/test_show.py new file mode 100644 index 00000000000..ac6aa8532a6 --- /dev/null +++ b/tests/integration/tui/test_show.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cylc.flow.exceptions import ClientError +from cylc.flow.tui.data import _show + + +async def test_show(flow, scheduler, start, raikura, monkeypatch): + """Test "cylc show" support in Tui.""" + id_ = flow({ + 'scheduling': { + 'graph': { + 'R1': 'foo' + }, + }, + 'runtime': { + 'foo': { + 'meta': { + 'title': 'Foo', + 'description': 'The first metasyntactic variable.' + }, + }, + }, + }, name='one') + schd = scheduler(id_) + async with start(schd): + await schd.update_data_structure() + + with raikura(size='80,40') as rk: + rk.user_input('down', 'right') + rk.wait_until_loaded(schd.tokens.id) + + # select a task + rk.user_input('down', 'down', 'enter') + + # select the "show" context option + rk.user_input(*(['down'] * 6), 'enter') + rk.compare_screenshot( + 'success', + 'the show output should be displayed', + ) + + # make it look like "cylc show" failed + def cli_cmd_fail(*args, **kwargs): + raise ClientError(':(') + monkeypatch.setattr( + 'cylc.flow.tui.data.cli_cmd', + cli_cmd_fail, + ) + + # select the "show" context option + rk.user_input('q', 'enter', *(['down'] * 6), 'enter') + rk.compare_screenshot( + 'fail', + 'the error should be displayed', + ) diff --git a/tests/integration/tui/test_updater.py b/tests/integration/tui/test_updater.py new file mode 100644 index 00000000000..39a9a606eff --- /dev/null +++ b/tests/integration/tui/test_updater.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import asyncio +from copy import deepcopy +from pathlib import Path +import re + +from async_timeout import timeout + +from cylc.flow.cycling.integer import IntegerPoint +from cylc.flow.id import Tokens +from cylc.flow.tui.updater import ( + Updater, + get_default_filters, +) +from cylc.flow.workflow_status import WorkflowStatus + + +async def await_update(updater): + """Wait for the latest update from the Updater. + + Note: + If multiple updates are waiting, this returns the most recent.Q + + Returns: + The latest update from the Updater's "update_queue". + + """ + while updater.update_queue.empty(): + await asyncio.sleep(0.1) + while not updater.update_queue.empty(): + # flush out any older updates to avoid race conditions in tests + update = updater.update_queue.get() + return update + + +async def wait_workflow_connected(updater, tokens, connected=True, time=3): + """Wait for the Updater to connect to a workflow. + + This will return once the updater has connected to a workflow and returned + the first data from it. + + Arugments: + tokens: + The tokens of the workflow you're waiting for. + connected: + If True this waits for the updater to connect to the workflow, if + False it waits for the updater to disconnect from it. + time: + The maximum time to wait for this to happen. + + Returns: + The first update from the Updater which contains the workflow data. + + """ + async with timeout(time): + while True: + root_node = await await_update(updater) + workflow_node = root_node['children'][0] + for workflow_node in root_node['children']: + if ( + workflow_node['id_'] == tokens.id + and ( + workflow_node['children'][0]['id_'] != '#spring' + ) == connected + ): + # if the spring node is still there then we haven't + # recieved the first update from the workflow yet + return root_node + + +def get_child_tokens(root_node, types, relative=False): + """Return all ID of the specified types contained within the provided tree. + + Args: + root_node: + The Tui tree you want to look for IDs in. + types: + The Tui types (e.g. 'workflow' or 'task') you want to extract. + relative: + If True, the relative IDs will be returned. + + """ + ret = set() + stack = [root_node] + while stack: + node = stack.pop() + stack.extend(node['children']) + if node['type_'] in types: + + tokens = Tokens(node['id_']) + if relative: + ret.add(tokens.relative_id) + else: + ret.add(tokens.id) + return ret + + +async def test_subscribe(one_conf, flow, scheduler, run, test_dir): + """It should subscribe and unsubscribe from workflows.""" + id_ = flow(one_conf) + schd = scheduler(id_) + + updater = Updater() + + async def the_test(): + nonlocal updater + + try: + # wait for the first update + root_node = await await_update(updater) + + # there should be a root root_node + assert root_node['id_'] == 'root' + # a single root_node representing the workflow + assert root_node['children'][0]['id_'] == schd.tokens.id + # and a "spring" root_node used to active the subscription + # mechanism + assert root_node['children'][0]['children'][0]['id_'] == '#spring' + + # subscribe to the workflow + updater.subscribe(schd.tokens.id) + + # wait for it to connect to the workflow + root_node = await wait_workflow_connected(updater, schd.tokens) + + # check the workflow contains one cycle with one task in it + workflow_node = root_node['children'][0] + assert len(workflow_node['children']) == 1 + cycle_node = workflow_node['children'][0] + assert Tokens(cycle_node['id_']).relative_id == '1' # cycle ID + assert len(cycle_node['children']) == 1 + task_node = cycle_node['children'][0] + assert Tokens(task_node['id_']).relative_id == '1/one' # task ID + + # unsubscribe from the workflow + updater.unsubscribe(schd.tokens.id) + + # wait for it to disconnect from the workflow + root_node = await wait_workflow_connected( + updater, + schd.tokens, + connected=False, + ) + + finally: + # shut down the updater + updater.terminate() + + async with run(schd): + filters = get_default_filters() + filters['workflows']['id'] = f'{re.escape(str(test_dir.relative_to(Path("~/cylc-run").expanduser())))}/.*' + + # run the updater and the test + async with timeout(10): + await asyncio.gather( + asyncio.create_task(updater.run(filters)), + asyncio.create_task(the_test()), + ) + + +async def test_filters(one_conf, flow, scheduler, run, test_dir): + """It should filter workflow and task states. + + Note: + The workflow ID filter is not explicitly tested here, but it is + indirectly tested, otherwise other workflows would show up in the + updater results. + + """ + one = scheduler(flow({ + 'scheduler': { + 'allow implicit tasks': 'True', + }, + 'scheduling': { + 'graph': { + 'R1': 'a & b & c', + } + } + }, name='one'), paused_start=True) + two = scheduler(flow(one_conf, name='two')) + tre = scheduler(flow(one_conf, name='tre')) + + filters = get_default_filters() + id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser())) + filters['workflows']['id'] = f'^{re.escape(id_base)}/.*' + + updater = Updater() + + async def the_test(): + nonlocal filters + try: + root_node = await await_update(updater) + assert {child['id_'] for child in root_node['children']} == { + one.tokens.id, + two.tokens.id, + tre.tokens.id, + } + + # filter out paused workflows + filters = deepcopy(filters) + filters['workflows'][WorkflowStatus.STOPPED.value] = True + filters['workflows'][WorkflowStatus.PAUSED.value] = False + updater.update_filters(filters) + + # "one" and "two" should now be filtered out + root_node = await await_update(updater) + assert {child['id_'] for child in root_node['children']} == { + tre.tokens.id, + } + + # filter out stopped workflows + filters = deepcopy(filters) + filters['workflows'][WorkflowStatus.STOPPED.value] = False + filters['workflows'][WorkflowStatus.PAUSED.value] = True + updater.update_filters(filters) + + # "tre" should now be filtered out + root_node = await await_update(updater) + assert {child['id_'] for child in root_node['children']} == { + one.tokens.id, + two.tokens.id, + } + + # subscribe to "one" + updater.subscribe(one.tokens.id) + root_node = await wait_workflow_connected(updater, one.tokens) + assert get_child_tokens( + root_node, types={'task'}, relative=True + ) == { + '1/a', + '1/b', + '1/c', + } + + # filter out running tasks + # TODO: see https://github.com/cylc/cylc-flow/issues/5716 + # filters = deepcopy(filters) + # filters['tasks'][TASK_STATUS_RUNNING] = False + # updater.update_filters(filters) + + # root_node = await await_update(updater) + # assert get_child_tokens( + # root_node, + # types={'task'}, + # relative=True + # ) == { + # '1/b', + # '1/c', + # } + + finally: + # shut down the updater + updater.terminate() + + # start workflow "one" + async with run(one): + # mark "1/a" as running and "1/b" as succeeded + one_a = one.pool.get_task(IntegerPoint('1'), 'a') + one_a.state_reset('running') + one.data_store_mgr.delta_task_state(one_a) + one.pool.get_task(IntegerPoint('1'), 'b').state_reset('succeeded') + + # start workflow "two" + async with run(two): + # run the updater and the test + async with timeout(10): + await asyncio.gather( + asyncio.create_task(the_test()), + asyncio.create_task(updater.run(filters)), + ) diff --git a/tests/unit/cfgspec/test_globalcfg.py b/tests/unit/cfgspec/test_globalcfg.py index 2becda1caad..6db8d76dcf8 100644 --- a/tests/unit/cfgspec/test_globalcfg.py +++ b/tests/unit/cfgspec/test_globalcfg.py @@ -148,3 +148,13 @@ def test_source_dir_validation( assert "must be an absolute path" in str(excinfo.value) else: glblcfg.load() + +def test_platform_ssh_forward_variables(mock_global_config): + + glblcfg: GlobalConfig = mock_global_config(''' + [platforms] + [[foo]] + ssh forward environment variables = "FOO", "BAR" + ''') + + assert glblcfg.get(['platforms','foo','ssh forward environment variables']) == ["FOO", "BAR"] diff --git a/tests/unit/cycling/test_iso8601.py b/tests/unit/cycling/test_iso8601.py index 98fc95b9c5b..ae0eb957f47 100644 --- a/tests/unit/cycling/test_iso8601.py +++ b/tests/unit/cycling/test_iso8601.py @@ -14,9 +14,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import pytest from datetime import datetime +import pytest +from pytest import param + from cylc.flow.cycling.iso8601 import ( ISO8601Interval, ISO8601Point, @@ -671,74 +673,52 @@ def test_simple(set_cycling_type): assert not sequence.is_on_sequence(ISO8601Point("20100809T0005")) -def test_next_simple(set_cycling_type): +@pytest.mark.parametrize( + 'value, expected', [ + ('next(T2100Z)', '20100808T2100Z'), + ('next(T00)', '20100809T0000Z'), + ('next(T-15)', '20100808T1615Z'), + ('next(T-45)', '20100808T1545Z'), + ('next(-10)', '21100101T0000Z'), + ('next(-1008)', '21100801T0000Z'), + ('next(--10)', '20101001T0000Z'), + ('next(--0325)', '20110325T0000Z'), + ('next(---10)', '20100810T0000Z'), + ('next(---05T1200Z)', '20100905T1200Z'), + param('next(--08-08)', '20110808T0000Z', marks=pytest.mark.xfail), + ('next(T15)', '20100809T1500Z'), + ('next(T-41)', '20100808T1541Z'), + ] +) +def test_next_simple(value: str, expected: str, set_cycling_type): """Test the generation of CP using 'next' from single input.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") - my_now = "20100808T1540Z" - sequence = ( - "next(T2100Z)", # 20100808T2100Z - "next(T00)", # 20100809T0000Z - "next(T-15)", # 20100808T1615Z - "next(T-45)", # 20100808T1545Z - "next(-10)", # 21100101T0000Z - "next(-1008)", # 21100801T0000Z - "next(--10)", # 20101001T0000Z - "next(--0325)", # 20110325T0000Z - "next(---10)", # 20100810T0000Z - "next(---05T1200Z)", # 20100905T1200Z - ) + my_now = "2010-08-08T15:41Z" + assert ingest_time(value, my_now) == expected - output = [] - for point in sequence: - output.append(ingest_time(point, my_now)) - assert output == [ - "20100808T2100Z", - "20100809T0000Z", - "20100808T1615Z", - "20100808T1545Z", - "21100101T0000Z", - "21100801T0000Z", - "20101001T0000Z", - "20110325T0000Z", - "20100810T0000Z", - "20100905T1200Z", +@pytest.mark.parametrize( + 'value, expected', [ + ('previous(T2100Z)', '20100807T2100Z'), + ('previous(T00)', '20100808T0000Z'), + ('previous(T-15)', '20100808T1515Z'), + ('previous(T-45)', '20100808T1445Z'), + ('previous(-10)', '20100101T0000Z'), + ('previous(-1008)', '20100801T0000Z'), + ('previous(--10)', '20091001T0000Z'), + ('previous(--0325)', '20100325T0000Z'), + ('previous(---10)', '20100710T0000Z'), + ('previous(---05T1200Z)', '20100805T1200Z'), + param('previous(--08-08)', '20100808T0000Z', marks=pytest.mark.xfail), + ('previous(T15)', '20100808T1500Z'), + ('previous(T-41)', '20100808T1441Z'), ] - - -def test_previous_simple(set_cycling_type): +) +def test_previous_simple(value: str, expected: str, set_cycling_type): """Test the generation of CP using 'previous' from single input.""" set_cycling_type(ISO8601_CYCLING_TYPE, "Z") - my_now = "20100808T1540Z" - sequence = ( - "previous(T2100Z)", # 20100807T2100Z - "previous(T00)", # 20100808T0000Z - "previous(T-15)", # 20100808T1515Z - "previous(T-45)", # 20100808T1445Z - "previous(-10)", # 20100101T0000Z - "previous(-1008)", # 20100801T0000Z - "previous(--10)", # 20091001T0000Z - "previous(--0325)", # 20100325T0000Z - "previous(---10)", # 20100710T0000Z - "previous(---05T1200Z)", # 20100805T1200Z - ) - - output = [] - - for point in sequence: - output.append(ingest_time(point, my_now)) - assert output == [ - "20100807T2100Z", - "20100808T0000Z", - "20100808T1515Z", - "20100808T1445Z", - "20100101T0000Z", - "20100801T0000Z", - "20091001T0000Z", - "20100325T0000Z", - "20100710T0000Z", - "20100805T1200Z", - ] + my_now = "2010-08-08T15:41Z" + assert ingest_time(value, my_now) == expected def test_sequence(set_cycling_type): @@ -855,63 +835,40 @@ def test_weeks_days(set_cycling_type): ] -def test_cug(set_cycling_type): - """Test the offset CP examples in the Cylc user guide""" +@pytest.mark.parametrize( + 'value, expected', [ + ('next(T-00)', '20180314T1600Z'), + ('previous(T-00)', '20180314T1500Z'), + ('next(T-00; T-15; T-30; T-45)', '20180314T1515Z'), + ('previous(T-00; T-15; T-30; T-45)', '20180314T1500Z'), + ('next(T00)', '20180315T0000Z'), + ('previous(T00)', '20180314T0000Z'), + ('next(T06:30Z)', '20180315T0630Z'), + ('previous(T06:30) -P1D', '20180313T0630Z'), + ('next(T00; T06; T12; T18)', '20180314T1800Z'), + ('previous(T00; T06; T12; T18)', '20180314T1200Z'), + ('next(T00; T06; T12; T18)+P1W', '20180321T1800Z'), + ('PT1H', '20180314T1612Z'), + ('-P1M', '20180214T1512Z'), + ('next(-00)', '21000101T0000Z'), + ('previous(--01)', '20180101T0000Z'), + ('next(---01)', '20180401T0000Z'), + ('previous(--1225)', '20171225T0000Z'), + ('next(-2006)', '20200601T0000Z'), + ('previous(-W101)', '20180305T0000Z'), + ('next(-W-1; -W-3; -W-5)', '20180314T0000Z'), + ('next(-001; -091; -181; -271)', '20180401T0000Z'), + ('previous(-365T12Z)', '20171231T1200Z'), + ] +) +def test_user_guide_examples(value: str, expected: str, set_cycling_type): + """Test the offset CP examples in the Cylc user guide. + + https://cylc.github.io/cylc-doc/stable/html/user-guide/writing-workflows/scheduling.html + """ set_cycling_type(ISO8601_CYCLING_TYPE, "Z") my_now = "2018-03-14T15:12Z" - sequence = ( - "next(T-00)", # 20180314T1600Z - "previous(T-00)", # 20180314T1500Z - "next(T-00; T-15; T-30; T-45)", # 20180314T1515Z - "previous(T-00; T-15; T-30; T-45)", # 20180314T1500Z - "next(T00)", # 20180315T0000Z - "previous(T00)", # 20180314T0000Z - "next(T06:30Z)", # 20180315T0630Z - "previous(T06:30) -P1D", # 20180313T0630Z - "next(T00; T06; T12; T18)", # 20180314T1800Z - "previous(T00; T06; T12; T18)", # 20180314T1200Z - "next(T00; T06; T12; T18)+P1W", # 20180321T1800Z - "PT1H", # 20180314T1612Z - "-P1M", # 20180214T1512Z - "next(-00)", # 21000101T0000Z - "previous(--01)", # 20180101T0000Z - "next(---01)", # 20180401T0000Z - "previous(--1225)", # 20171225T0000Z - "next(-2006)", # 20200601T0000Z - "previous(-W101)", # 20180305T0000Z - "next(-W-1; -W-3; -W-5)", # 20180314T0000Z - "next(-001; -091; -181; -271)", # 20180401T0000Z - "previous(-365T12Z)", # 20171231T1200Z - ) - - output = [] - - for point in sequence: - output.append(ingest_time(point, my_now)) - assert output == [ - "20180314T1600Z", - "20180314T1500Z", - "20180314T1515Z", - "20180314T1500Z", - "20180315T0000Z", - "20180314T0000Z", - "20180315T0630Z", - "20180313T0630Z", - "20180314T1800Z", - "20180314T1200Z", - "20180321T1800Z", - "20180314T1612Z", - "20180214T1512Z", - "21000101T0000Z", - "20180101T0000Z", - "20180401T0000Z", - "20171225T0000Z", - "20200601T0000Z", - "20180305T0000Z", - "20180314T0000Z", - "20180401T0000Z", - "20171231T1200Z", - ] + assert ingest_time(value, my_now) == expected def test_next_simple_no_now(set_cycling_type): diff --git a/tests/unit/plugins/test_pre_configure.py b/tests/unit/plugins/test_pre_configure.py index 717648ea594..e9f91054e12 100644 --- a/tests/unit/plugins/test_pre_configure.py +++ b/tests/unit/plugins/test_pre_configure.py @@ -31,7 +31,7 @@ def __init__(self, fcn): self.name = fcn.__name__ self.fcn = fcn - def resolve(self): + def load(self): return self.fcn diff --git a/tests/unit/scripts/test_completion_server.py b/tests/unit/scripts/test_completion_server.py index d50e46b7039..186e13b7272 100644 --- a/tests/unit/scripts/test_completion_server.py +++ b/tests/unit/scripts/test_completion_server.py @@ -398,18 +398,17 @@ def test_list_options(monkeypatch): assert list_options('zz9+za') == [] # patch the logic to turn off the auto_add behaviour of CylcOptionParser - def _resolve(): - def _parser_function(): - parser = get_option_parser() - del parser.auto_add - return parser - - return SimpleNamespace(parser_function=_parser_function) - - monkeypatch.setattr( - COMMANDS['trigger'], - 'resolve', - _resolve + class EntryPoint: + def load(self): + def _parser_function(): + parser = get_option_parser() + del parser.auto_add + return parser + return SimpleNamespace(parser_function=_parser_function) + monkeypatch.setitem( + COMMANDS, + 'trigger', + EntryPoint(), ) # with auto_add turned off the --color option should be absent @@ -674,7 +673,7 @@ def _get_current_completion_script_version(_script, lang): # set the completion script compatibility range to >=1.0.0, <2.0.0 monkeypatch.setattr( 'cylc.flow.scripts.completion_server.REQUIRED_SCRIPT_VERSION', - 'completion-script >=1.0.0, <2.0.0', + '>=1.0.0, <2.0.0', ) monkeypatch.setattr( 'cylc.flow.scripts.completion_server' diff --git a/tests/unit/scripts/test_cylc.py b/tests/unit/scripts/test_cylc.py index f1483e9ee4a..819583a296c 100644 --- a/tests/unit/scripts/test_cylc.py +++ b/tests/unit/scripts/test_cylc.py @@ -15,14 +15,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import pkg_resources +import os +import sys from types import SimpleNamespace from typing import Callable from unittest.mock import Mock import pytest -from cylc.flow.scripts.cylc import iter_commands +from cylc.flow.scripts.cylc import iter_commands, pythonpath_manip from ..conftest import MonkeyMock @@ -30,12 +31,9 @@ @pytest.fixture def mock_entry_points(monkeypatch: pytest.MonkeyPatch): """Mock a range of entry points.""" - def _resolve_fail(*args, **kwargs): + def _load_fail(*args, **kwargs): raise ModuleNotFoundError('foo') - def _require_fail(*args, **kwargs): - raise pkg_resources.DistributionNotFound('foo', ['my_extras']) - def _resolve_ok(*args, **kwargs): return Mock() @@ -47,24 +45,18 @@ def _mocked_entry_points(include_bad: bool = False): # an entry point with all dependencies installed: 'good': SimpleNamespace( name='good', - module_name='os.path', - resolve=_resolve_ok, - require=_require_ok, + module='os.path', + load=_resolve_ok, + extras=[], + dist=SimpleNamespace(name='a'), ), # an entry point with optional dependencies missing: 'missing': SimpleNamespace( name='missing', - module_name='not.a.python.module', # force an import error - resolve=_resolve_fail, - require=_require_fail, - ), - # an entry point with optional dependencies missing, but they - # are not needed for the core functionality of the entry point: - 'partial': SimpleNamespace( - name='partial', - module_name='os.path', - resolve=_resolve_ok, - require=_require_fail, + module='not.a.python.module', # force an import error + load=_load_fail, + extras=[], + dist=SimpleNamespace(name='foo'), ), } if include_bad: @@ -72,9 +64,11 @@ def _mocked_entry_points(include_bad: bool = False): # missing: commands['bad'] = SimpleNamespace( name='bad', - module_name='not.a.python.module', - resolve=_resolve_fail, + module='not.a.python.module', + load=_load_fail, require=_require_ok, + extras=[], + dist=SimpleNamespace(name='d'), ) monkeypatch.setattr('cylc.flow.scripts.cylc.COMMANDS', commands) @@ -88,14 +82,13 @@ def test_iter_commands(mock_entry_points): """ mock_entry_points() commands = list(iter_commands()) - assert [i[0] for i in commands] == ['good', 'partial'] + assert [i[0] for i in commands] == ['good'] def test_iter_commands_bad(mock_entry_points): - """Test listing commands fails if there is an unexpected import error.""" + """Test listing commands doesn't fail on import error.""" mock_entry_points(include_bad=True) - with pytest.raises(ModuleNotFoundError): - list(iter_commands()) + list(iter_commands()) def test_execute_cmd( @@ -123,16 +116,35 @@ def test_execute_cmd( execute_cmd('missing') capexit.assert_any_call(1) assert capsys.readouterr().err.strip() == ( - "cylc missing: The 'foo' distribution was not found and is" - " required by my_extras" + '"cylc missing" requires "foo"\n\nModuleNotFoundError: foo' ) - # the "partial" entry point should exit 0 - capexit.reset_mock() - execute_cmd('partial') - capexit.assert_called_once_with() - assert capsys.readouterr().err == '' + # the "bad" entry point should log an error + execute_cmd('bad') + capexit.assert_any_call(1) + + stderr = capsys.readouterr().err.strip() + assert '"cylc bad" requires "d"' in stderr + assert 'ModuleNotFoundError: foo' in stderr + - # the "bad" entry point should raise an exception - with pytest.raises(ModuleNotFoundError): - execute_cmd('bad') +def test_pythonpath_manip(monkeypatch): + """pythonpath_manip removes items in PYTHONPATH from sys.path + + and adds items from CYLC_PYTHONPATH + """ + # If PYTHONPATH is set... + monkeypatch.setenv('PYTHONPATH', '/remove-from-sys.path') + monkeypatch.setattr('sys.path', ['/leave-alone', '/remove-from-sys.path']) + pythonpath_manip() + # ... we don't change PYTHONPATH + assert os.environ['PYTHONPATH'] == '/remove-from-sys.path' + # ... but we do remove PYTHONPATH items from sys.path, and don't remove + # items there not in PYTHONPATH + assert sys.path == ['/leave-alone'] + + # If CYLC_PYTHONPATH is set we retrieve its contents and + # add them to the sys.path: + monkeypatch.setenv('CYLC_PYTHONPATH', '/add-to-sys.path') + pythonpath_manip() + assert sys.path == ['/add-to-sys.path', '/leave-alone'] diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py index 205d09019fa..6ee1018cf96 100644 --- a/tests/unit/scripts/test_lint.py +++ b/tests/unit/scripts/test_lint.py @@ -28,11 +28,10 @@ MANUAL_DEPRECATIONS, get_cylc_files, get_pyproject_toml, - get_reference_rst, - get_reference_text, + get_reference, get_upgrader_info, lint, - merge_cli_with_tomldata, + _merge_cli_with_tomldata, parse_checks, validate_toml_items ) @@ -103,6 +102,7 @@ pre-script = "echo ${CYLC_SUITE_DEF_PATH}" script = {{HELLOWORLD}} post-script = "echo ${CYLC_SUITE_INITIAL_CYCLE_TIME}" + env-script = POINT=$(rose date 2059 --offset P1M) [[[suite state polling]]] template = and [[[remote]]] @@ -142,7 +142,6 @@ [[and_another_thing]] [[[remote]]] host = `rose host-select thingy` - """ @@ -159,6 +158,9 @@ # {{quix}} [runtime] + [[this_is_ok]] + script = echo "this is incorrectly indented" + [[foo]] inherit = hello [[[job]]] @@ -332,6 +334,12 @@ def test_check_cylc_file_jinja2_comments(): assert not any('S011' in msg for msg in lint.messages) +def test_check_cylc_file_jinja2_comments_shell_arithmetic_not_warned(): + """Jinja2 after a $((10#$variable)) should not warn""" + lint = lint_text('#!jinja2\na = b$((10#$foo+5)) {{ BAR }}', ['style']) + assert not any('S011' in msg for msg in lint.messages) + + @pytest.mark.parametrize( # 11 won't be tested because there is no jinja2 shebang 'number', set(range(1, len(MANUAL_DEPRECATIONS) + 1)) - {11} @@ -359,35 +367,47 @@ def test_get_cylc_files_get_all_rcs(tmp_path): assert sorted(result) == sorted(expect) -MOCK_CHECKS = { - 'U042': { - 'short': 'section `[vizualization]` has been removed.', - 'url': 'some url or other', - 'purpose': 'U', - 'rst': 'section ``[vizualization]`` has been removed.', - 'function': re.compile('not a regex') - }, -} +def mock_parse_checks(*args, **kwargs): + return { + 'U042': { + 'short': 'section `[vizualization]` has been removed.', + 'url': 'some url or other', + 'purpose': 'U', + 'rst': 'section ``[vizualization]`` has been removed.', + 'function': re.compile('not a regex') + }, + } -def test_get_reference_rst(): +def test_get_reference_rst(monkeypatch): """It produces a reference file for our linting.""" - ref = get_reference_rst(MOCK_CHECKS) + monkeypatch.setattr( + 'cylc.flow.scripts.lint.parse_checks', mock_parse_checks + ) + ref = get_reference('all', 'rst') expect = ( '\n7 to 8 upgrades\n---------------\n\n' - 'U042\n^^^^\nsection ``[vizualization]`` has been ' + '`U042 `_' + f'\n{ "^" * 78 }' + '\nsection ``[vizualization]`` has been ' 'removed.\n\n\n' ) assert ref == expect -def test_get_reference_text(): +def test_get_reference_text(monkeypatch): """It produces a reference file for our linting.""" - ref = get_reference_text(MOCK_CHECKS) + monkeypatch.setattr( + 'cylc.flow.scripts.lint.parse_checks', mock_parse_checks + ) + ref = get_reference('all', 'text') expect = ( '\n7 to 8 upgrades\n---------------\n\n' 'U042:\n section `[vizualization]` has been ' - 'removed.\n\n\n' + 'removed.' + '\n https://cylc.github.io/cylc-doc/stable/html/7-to-8/some' + ' url or other\n\n\n' ) assert ref == expect @@ -556,7 +576,7 @@ def test_validate_toml_items(input_, error): ) def test_merge_cli_with_tomldata(clidata, tomldata, expect): """It merges each of the three sections correctly: see function.__doc__""" - assert merge_cli_with_tomldata(clidata, tomldata) == expect + assert _merge_cli_with_tomldata(clidata, tomldata) == expect def test_invalid_tomlfile(tmp_path): @@ -572,11 +592,45 @@ def test_invalid_tomlfile(tmp_path): 'ref, expect', [ [True, 'line > ```` characters'], - [False, 'line > 130 characters'] + [False, 'line > 42 characters'] ] ) def test_parse_checks_reference_mode(ref, expect): - result = parse_checks(['style'], reference=ref) - key = list(result.keys())[-1] - value = result[key] + """Add extra explanation of max line legth setting in reference mode. + """ + result = parse_checks(['style'], reference=ref, max_line_len=42) + value = result['S012'] assert expect in value['short'] + + +@pytest.mark.parametrize( + 'spaces, expect', + ( + (0, 'S002'), + (1, 'S013'), + (2, 'S013'), + (3, 'S013'), + (4, None), + (5, 'S013'), + (6, 'S013'), + (7, 'S013'), + (8, None), + (9, 'S013') + ) +) +def test_indents(spaces, expect): + """Test different wrong indentations + + Parameterization deliberately over-obvious to avoid replicating + arithmetic logic from code. Dangerously close to re-testing ``%`` + builtin. + """ + result = lint_text( + f"{' ' * spaces}foo = 42", + ['style'] + ) + result = ''.join(result.messages) + if expect: + assert expect in result + else: + assert not result diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 34c596a770c..bb55cbf295e 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from copy import deepcopy import os import sys from optparse import Values @@ -38,12 +39,13 @@ from cylc.flow.parsec.exceptions import Jinja2Error, EmPyError from cylc.flow.scheduler_cli import RunOptions from cylc.flow.scripts.validate import ValidateOptions +from cylc.flow.simulation import configure_sim_modes from cylc.flow.workflow_files import WorkflowFiles from cylc.flow.wallclock import get_utc_mode, set_utc_mode from cylc.flow.xtrigger_mgr import XtriggerManager from cylc.flow.task_outputs import ( TASK_OUTPUT_SUBMITTED, - TASK_OUTPUT_SUCCEEDED + TASK_OUTPUT_SUCCEEDED, ) from cylc.flow.cycling.iso8601 import ISO8601Point @@ -1741,3 +1743,31 @@ def test_cylc_env_at_parsing( assert var in cylc_env else: assert var not in cylc_env + + +def test_configure_sim_mode(caplog): + job_section = {} + sim_section = { + 'speedup factor': '', + 'default run length': 'PT10S', + 'time limit buffer': 'PT0S', + 'fail try 1 only': False, + 'fail cycle points': '', + } + rtconfig_1 = { + 'execution time limit': '', + 'simulation': sim_section, + 'job': job_section, + 'outputs': {}, + } + rtconfig_2 = deepcopy(rtconfig_1) + rtconfig_2['simulation']['default run length'] = 'PT2S' + + taskdefs = [ + SimpleNamespace(rtconfig=rtconfig_1), + SimpleNamespace(rtconfig=rtconfig_2), + ] + configure_sim_modes(taskdefs, 'simulation') + results = [ + i.rtconfig['simulation']['simulated run length'] for i in taskdefs] + assert results == [10.0, 2.0] diff --git a/tests/unit/test_graph_parser.py b/tests/unit/test_graph_parser.py index 97b7fb45483..ddd443a3597 100644 --- a/tests/unit/test_graph_parser.py +++ b/tests/unit/test_graph_parser.py @@ -16,6 +16,7 @@ """Unit tests for the GraphParser.""" import logging +from typing import Dict, List import pytest from itertools import product from pytest import param @@ -86,45 +87,59 @@ def test_graph_syntax_errors_2(seq, graph, expected_err): @pytest.mark.parametrize( 'graph, expected_err', [ - [ + ( "a b => c", "Bad graph node format" - ], - [ + ), + ( + "a => b c", + "Bad graph node format" + ), + ( "!foo => bar", "Suicide markers must be on the right of a trigger:" - ], - [ + ), + ( "( foo & bar => baz", - "Mismatched parentheses in:" - ], - [ + 'Mismatched parentheses in: "(foo&bar"' + ), + ( + "a => b & c)", + 'Mismatched parentheses in: "b&c)"' + ), + ( + "(a => b & c)", + 'Mismatched parentheses in: "(a"' + ), + ( + "(a => b[+P1]", + 'Mismatched parentheses in: "(a"' + ), + ( """(a | b & c) => d foo => bar (a | b & c) => !d""", "can't trigger both d and !d" - ], - [ + ), + ( "a => b | c", "Illegal OR on right side" - ], - [ + ), + ( "foo && bar => baz", "The graph AND operator is '&'" - ], - [ + ), + ( "foo || bar => baz", "The graph OR operator is '|'" - ], + ), ] ) def test_graph_syntax_errors(graph, expected_err): """Test various graph syntax errors.""" with pytest.raises(GraphParseError) as cm: GraphParser().parse_graph(graph) - assert ( - expected_err in str(cm.value) - ) + assert expected_err in str(cm.value) def test_parse_graph_simple(): @@ -845,3 +860,39 @@ def test_fail_family_triggers_on_tasks(ftrig): "family trigger on non-family namespace" ) ) + + +@pytest.mark.parametrize( + 'graph, expected_triggers', + [ + param( + 'a => b & c', + {'a': [''], 'b': ['a:succeeded'], 'c': ['a:succeeded']}, + id="simple" + ), + param( + 'a => (b & c)', + {'a': [''], 'b': ['a:succeeded'], 'c': ['a:succeeded']}, + id="simple w/ parentheses" + ), + param( + 'a => (b & (c & d))', + { + 'a': [''], + 'b': ['a:succeeded'], + 'c': ['a:succeeded'], + 'd': ['a:succeeded'], + }, + id="more parentheses" + ), + ] +) +def test_RHS_AND(graph: str, expected_triggers: Dict[str, List[str]]): + """Test '&' operator on right hand side of trigger expression.""" + gp = GraphParser() + gp.parse_graph(graph) + triggers = { + task: list(trigs.keys()) + for task, trigs in gp.triggers.items() + } + assert triggers == expected_triggers diff --git a/tests/unit/test_id_cli.py b/tests/unit/test_id_cli.py index 149990a9bb0..9642a3c9874 100644 --- a/tests/unit/test_id_cli.py +++ b/tests/unit/test_id_cli.py @@ -35,9 +35,8 @@ @pytest.fixture -def mock_exists(mocker): - mock_exists = mocker.patch('pathlib.Path.exists') - mock_exists.return_value = True +def mock_exists(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr('pathlib.Path.exists', lambda *a, **k: True) @pytest.fixture(scope='module') diff --git a/tests/unit/test_indep_task_queues.py b/tests/unit/test_indep_task_queues.py index a0a1894cece..d144f58eecf 100644 --- a/tests/unit/test_indep_task_queues.py +++ b/tests/unit/test_indep_task_queues.py @@ -21,8 +21,8 @@ import pytest +from cylc.flow.task_proxy import TaskProxy from cylc.flow.task_queues.independent import IndepQueueManager -from cylc.flow.task_state import TASK_STATUS_PREPARING MEMBERS = {"a", "b", "c", "d", "e", "f"} @@ -61,9 +61,7 @@ @pytest.mark.parametrize( - "active," - "expected_released," - "expected_foo_groups", + "active, expected_released, expected_foo_groups", [ ( Counter(["b1", "b2", "s1", "o1"]), @@ -73,28 +71,24 @@ ] ) def test_queue_and_release( - active, - expected_released, - expected_foo_groups): + active, + expected_released, + expected_foo_groups +): """Test task queue and release.""" # configure the queue queue_mgr = IndepQueueManager(QCONFIG, ALL_TASK_NAMES, DESCENDANTS) # add newly ready tasks to the queue for name in READY_TASK_NAMES: - itask = Mock() + itask = Mock(spec=TaskProxy) itask.tdef.name = name itask.state.is_held = False queue_mgr.push_task(itask) # release tasks, given current active task counter released = queue_mgr.release_tasks(active) - assert sorted([r.tdef.name for r in released]) == sorted(expected_released) - - # check released tasks change state to "preparing", and not is_queued - for r in released: - assert r.state.reset.called_with(TASK_STATUS_PREPARING) - assert r.state.reset.called_with(is_queued=False) + assert sorted(r.tdef.name for r in released) == sorted(expected_released) # check that adopted orphans end up in the default queue orphans = ["orphan1", "orphan2"] diff --git a/tests/unit/test_job_file.py b/tests/unit/test_job_file.py index 92be2d167c1..9dbfa58f3a1 100644 --- a/tests/unit/test_job_file.py +++ b/tests/unit/test_job_file.py @@ -31,7 +31,6 @@ ) from cylc.flow.job_file import ( JobFileWriter, - MAX_CYLC_TASK_DEPENDENCIES_LEN, ) from cylc.flow.platforms import platform_from_name @@ -398,7 +397,6 @@ def test_write_task_environment(): 'export CYLC_TASK_COMMS_METHOD=ssh\n ' 'export CYLC_TASK_JOB="1/moo/01"\n export ' 'CYLC_TASK_NAMESPACE_HIERARCHY="baa moo"\n export ' - 'CYLC_TASK_DEPENDENCIES="moo neigh quack"\n export ' 'CYLC_TASK_TRY_NUMBER=1\n export ' 'CYLC_TASK_FLOW_NUMBERS=1\n export ' 'CYLC_TASK_PARAM_duck="quack"\n export ' @@ -537,46 +535,3 @@ def test_homeless_platform(fixture_get_platform): if 'HOME' in job_sh_txt: raise Exception('$HOME found in job.sh\n{job_sh_txt}') - -def test_cylc_task_dependencies_length(): - f"""Test CYLC_TASK_DEPENDENCIES variable toggle. - - The CYLC_TASK_DEPENDENCIES variriable should only be exported if there are - { MAX_CYLC_TASK_DEPENDENCIES_LEN } or fewer dependencies. - - See: https://github.com/cylc/cylc-flow/issues/5551 - - """ - job_conf = { - 'platform': {'communication method': 'zmq'}, - 'job_d': 'a/b/c', - 'namespace_hierarchy': ['a', 'b'], - # the maximum permitted number of dependencies before the environment - # variable is omitted - 'dependencies': ['a'] * (MAX_CYLC_TASK_DEPENDENCIES_LEN), - 'try_num': 1, - 'flow_nums': {1}, - 'param_var': {}, - 'work_d': 'b/c/d', - } - - # write the job environment - with io.StringIO() as fake_file: - JobFileWriter()._write_task_environment(fake_file, job_conf) - output = fake_file.getvalue() - - # assert the env var is exported - lines = [line.strip().split('=')[0] for line in output.splitlines()] - assert 'export CYLC_TASK_DEPENDENCIES' in lines - - # add an extra dependency to push it over the limit - job_conf['dependencies'] += ['b'] - - # write the job environment - with io.StringIO() as fake_file: - JobFileWriter()._write_task_environment(fake_file, job_conf) - output = fake_file.getvalue() - - # assert the env var is redacted - lines = [line.strip().split('=')[0] for line in output.splitlines()] - assert '# CYLC_TASK_DEPENDENCIES' in lines # var should be commented out diff --git a/tests/unit/test_pipe_poller.py b/tests/unit/test_pipe_poller.py new file mode 100644 index 00000000000..a41e9635c6b --- /dev/null +++ b/tests/unit/test_pipe_poller.py @@ -0,0 +1,33 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from subprocess import Popen, PIPE + +from cylc.flow.pipe_poller import pipe_poller + + +def test_pipe_poller_str(): + proc = Popen(['echo', 'Hello World!'], stdout=PIPE, text=True) + (stdout,) = pipe_poller(proc, proc.stdout) + assert proc.returncode == 0 + assert stdout == 'Hello World!\n' + + +def test_pipe_poller_bytes(): + proc = Popen(['echo', 'Hello World!'], stdout=PIPE, text=False) + (stdout,) = pipe_poller(proc, proc.stdout) + assert proc.returncode == 0 + assert stdout == b'Hello World!\n' diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py index 7be01de2e20..5982fb746f1 100644 --- a/tests/unit/test_remote.py +++ b/tests/unit/test_remote.py @@ -15,7 +15,15 @@ # along with this program. If not, see . """Test the cylc.flow.remote module.""" -from cylc.flow.remote import run_cmd, construct_rsync_over_ssh_cmd +import os +from unittest import mock + +import pytest + +from cylc.flow.remote import ( + run_cmd, construct_rsync_over_ssh_cmd, construct_ssh_cmd +) +import cylc.flow def test_run_cmd_stdin_str(): @@ -86,3 +94,31 @@ def test_construct_rsync_over_ssh_cmd(): '/foo/', 'miklegard:/bar/', ] + + +def test_construct_ssh_cmd_forward_env(monkeypatch: pytest.MonkeyPatch): + """ Test for 'ssh forward environment variables' + """ + # Clear CYLC_* env vars as these will show up in the command + for env_var in os.environ: + if env_var.startswith('CYLC'): + monkeypatch.delenv(env_var) + + host = 'example.com' + config = { + 'ssh command': 'ssh', + 'use login shell': None, + 'cylc path': None, + 'ssh forward environment variables': ['FOO', 'BAZ'], + } + + # Variable isn't set, no change to command + expect = ['ssh', host, 'env', f'CYLC_VERSION={cylc.flow.__version__}', 'cylc', 'play'] + cmd = construct_ssh_cmd(['play'], config, host) + assert cmd == expect + + # Variable is set, appears in `env` list + monkeypatch.setenv('FOO', 'BAR') + expect = ['ssh', host, 'env', f'CYLC_VERSION={cylc.flow.__version__}', 'FOO=BAR', 'cylc', 'play'] + cmd = construct_ssh_cmd(['play'], config, host) + assert cmd == expect diff --git a/tests/unit/test_simulation.py b/tests/unit/test_simulation.py new file mode 100644 index 00000000000..1c490f35c16 --- /dev/null +++ b/tests/unit/test_simulation.py @@ -0,0 +1,166 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Tests for utilities supporting simulation and skip modes +""" +import pytest +from pytest import param + +from cylc.flow.cycling.integer import IntegerPoint +from cylc.flow.cycling.iso8601 import ISO8601Point +from cylc.flow.simulation import ( + parse_fail_cycle_points, + build_dummy_script, + disable_platforms, + get_simulated_run_len, + sim_task_failed, +) + + +@pytest.mark.parametrize( + 'execution_time_limit, speedup_factor, default_run_length', + ( + param(None, None, 'PT1H', id='default-run-length'), + param(None, 10, 'PT1H', id='speedup-factor-alone'), + param('PT1H', None, 'PT1H', id='execution-time-limit-alone'), + param('P1D', 24, 'PT1M', id='speed-up-and-execution-tl'), + ) +) +def test_get_simulated_run_len( + execution_time_limit, speedup_factor, default_run_length +): + """Test the logic of the presence or absence of config items. + + Avoid testing the correct workign of DurationParser. + """ + rtc = { + 'execution time limit': execution_time_limit, + 'simulation': { + 'speedup factor': speedup_factor, + 'default run length': default_run_length + } + } + assert get_simulated_run_len(rtc) == 3600 + + +@pytest.mark.parametrize( + 'fail_one_time_only', (True, False) +) +def test_set_simulation_script(fail_one_time_only): + rtc = { + 'outputs': {'foo': '1', 'bar': '2'}, + 'simulation': { + 'fail try 1 only': fail_one_time_only, + 'fail cycle points': '1', + } + } + result = build_dummy_script(rtc, 60) + assert result.split('\n') == [ + 'sleep 60', + "cylc message '1'", + "cylc message '2'", + f"cylc__job__dummy_result {str(fail_one_time_only).lower()}" + " 1 || exit 1" + ] + + +@pytest.mark.parametrize( + 'rtc, expect', ( + ({'platform': 'skarloey'}, 'localhost'), + ({'remote': {'host': 'rheneas'}}, 'localhost'), + ({'job': {'batch system': 'loaf'}}, 'localhost'), + ) +) +def test_disable_platforms(rtc, expect): + """A sampling of items FORBIDDEN_WITH_PLATFORMS are removed from a + config passed to this method. + """ + disable_platforms(rtc) + assert rtc['platform'] == expect + subdicts = [v for v in rtc.values() if isinstance(v, dict)] + for subdict in subdicts: + for k, val in subdict.items(): + if k != 'platform': + assert val is None + + +def test_parse_fail_cycle_points(set_cycling_type): + before = ['2', '4'] + set_cycling_type() + assert parse_fail_cycle_points(before) == [ + IntegerPoint(i) for i in before + ] + + +@pytest.mark.parametrize( + 'conf, point, try_, expect', + ( + param( + {'fail cycle points': [], 'fail try 1 only': True}, + ISO8601Point('1'), + 1, + False, + id='defaults' + ), + param( + {'fail cycle points': None, 'fail try 1 only': True}, + ISO8601Point('1066'), + 1, + True, + id='fail-all' + ), + param( + { + 'fail cycle points': [ + ISO8601Point('1066'), ISO8601Point('1067')], + 'fail try 1 only': True + }, + ISO8601Point('1067'), + 1, + True, + id='point-in-failCP' + ), + param( + { + 'fail cycle points': [ + ISO8601Point('1066'), ISO8601Point('1067')], + 'fail try 1 only': True + }, + ISO8601Point('1000'), + 1, + False, + id='point-notin-failCP' + ), + param( + {'fail cycle points': None, 'fail try 1 only': True}, + ISO8601Point('1066'), + 2, + False, + id='succeed-attempt2' + ), + param( + {'fail cycle points': None, 'fail try 1 only': False}, + ISO8601Point('1066'), + 7, + True, + id='fail-attempt7' + ), + ) +) +def test_sim_task_failed( + conf, point, try_, expect, set_cycling_type +): + set_cycling_type('iso8601') + assert sim_task_failed(conf, point, try_) == expect diff --git a/tests/unit/test_task_proxy.py b/tests/unit/test_task_proxy.py index 5369e70f124..98695ecd13f 100644 --- a/tests/unit/test_task_proxy.py +++ b/tests/unit/test_task_proxy.py @@ -60,9 +60,10 @@ def test_get_clock_trigger_time( set_cycling_type(itask_point.TYPE) mock_itask = Mock( point=itask_point.standardise(), - clock_trigger_time=None + clock_trigger_times={} ) - assert TaskProxy.get_clock_trigger_time(mock_itask, offset_str) == expected + assert TaskProxy.get_clock_trigger_time( + mock_itask, mock_itask.point, offset_str) == expected @pytest.mark.parametrize( diff --git a/tests/unit/tui/test_data.py b/tests/unit/tui/test_data.py index a2d17bf2e76..85805a5d1ea 100644 --- a/tests/unit/tui/test_data.py +++ b/tests/unit/tui/test_data.py @@ -28,7 +28,7 @@ def test_generate_mutation(monkeypatch): monkeypatch.setattr(cylc.flow.tui.data, 'ARGUMENT_TYPES', arg_types) assert generate_mutation( 'my_mutation', - ['foo', 'bar'] + {'foo': 'foo', 'bar': 'bar', 'user': 'user'} ) == ''' mutation($foo: String!, $bar: [Int]) { my_mutation (foos: $foo, bars: $bar) { diff --git a/tests/unit/tui/test_overlay.py b/tests/unit/tui/test_overlay.py index 42334aac009..013e8480c21 100644 --- a/tests/unit/tui/test_overlay.py +++ b/tests/unit/tui/test_overlay.py @@ -21,7 +21,9 @@ import pytest import urwid +from cylc.flow.tui.app import BINDINGS import cylc.flow.tui.overlay +from cylc.flow.workflow_status import WorkflowStatus @pytest.fixture @@ -39,6 +41,7 @@ def overlay_functions(): getattr(cylc.flow.tui.overlay, obj.name) for obj in tree.body if isinstance(obj, ast.FunctionDef) + and not obj.name.startswith('_') ] @@ -47,14 +50,21 @@ def test_interface(overlay_functions): for function in overlay_functions: # mock up an app object to keep things working app = Mock( - filter_states={}, + filters={'tasks': {}, 'workflows': {'id': '.*'}}, + bindings=BINDINGS, tree_walker=Mock( get_focus=Mock( return_value=[ Mock( get_node=Mock( return_value=Mock( - get_value=lambda: {'id_': 'a'} + get_value=lambda: { + 'id_': '~u/a', + 'type_': 'workflow', + 'data': { + 'status': WorkflowStatus.RUNNING, + }, + } ) ) ) diff --git a/tests/unit/tui/test_util.py b/tests/unit/tui/test_util.py index 00ac9fa95be..2b3231e0f7e 100644 --- a/tests/unit/tui/test_util.py +++ b/tests/unit/tui/test_util.py @@ -189,77 +189,87 @@ def test_compute_tree(): """ tree = compute_tree({ - 'id': 'workflow id', - 'cyclePoints': [ - { - 'id': '1/family-suffix', - 'cyclePoint': '1' - } - ], - 'familyProxies': [ - { # top level family - 'name': 'FOO', - 'id': '1/FOO', - 'cyclePoint': '1', - 'firstParent': {'name': 'root', 'id': '1/root'} - }, - { # nested family - 'name': 'FOOT', - 'id': '1/FOOT', - 'cyclePoint': '1', - 'firstParent': {'name': 'FOO', 'id': '1/FOO'} - }, - ], - 'taskProxies': [ - { # top level task - 'name': 'pub', - 'id': '1/pub', - 'firstParent': {'name': 'root', 'id': '1/root'}, - 'cyclePoint': '1', - 'jobs': [] - }, - { # child task (belongs to family) - 'name': 'fan', - 'id': '1/fan', - 'firstParent': {'name': 'fan', 'id': '1/fan'}, - 'cyclePoint': '1', - 'jobs': [] - }, - { # nested child task (belongs to incestuous family) - 'name': 'fool', - 'id': '1/fool', - 'firstParent': {'name': 'FOOT', 'id': '1/FOOT'}, - 'cyclePoint': '1', - 'jobs': [] - }, - { # a task which has jobs - 'name': 'worker', - 'id': '1/worker', - 'firstParent': {'name': 'root', 'id': '1/root'}, - 'cyclePoint': '1', - 'jobs': [ - {'id': '1/worker/03', 'submitNum': '3'}, - {'id': '1/worker/02', 'submitNum': '2'}, - {'id': '1/worker/01', 'submitNum': '1'} - ] - } - ] + 'workflows': [{ + 'id': 'workflow id', + 'port': 1234, + 'cyclePoints': [ + { + 'id': '1/family-suffix', + 'cyclePoint': '1' + } + ], + 'familyProxies': [ + { # top level family + 'name': 'FOO', + 'id': '1/FOO', + 'cyclePoint': '1', + 'firstParent': {'name': 'root', 'id': '1/root'} + }, + { # nested family + 'name': 'FOOT', + 'id': '1/FOOT', + 'cyclePoint': '1', + 'firstParent': {'name': 'FOO', 'id': '1/FOO'} + }, + ], + 'taskProxies': [ + { # top level task + 'name': 'pub', + 'id': '1/pub', + 'firstParent': {'name': 'root', 'id': '1/root'}, + 'cyclePoint': '1', + 'jobs': [] + }, + { # child task (belongs to family) + 'name': 'fan', + 'id': '1/fan', + 'firstParent': {'name': 'fan', 'id': '1/fan'}, + 'cyclePoint': '1', + 'jobs': [] + }, + { # nested child task (belongs to incestuous family) + 'name': 'fool', + 'id': '1/fool', + 'firstParent': {'name': 'FOOT', 'id': '1/FOOT'}, + 'cyclePoint': '1', + 'jobs': [] + }, + { # a task which has jobs + 'name': 'worker', + 'id': '1/worker', + 'firstParent': {'name': 'root', 'id': '1/root'}, + 'cyclePoint': '1', + 'jobs': [ + {'id': '1/worker/03', 'submitNum': '3'}, + {'id': '1/worker/02', 'submitNum': '2'}, + {'id': '1/worker/01', 'submitNum': '1'} + ] + } + ] + }] }) + # the root node + assert tree['type_'] == 'root' + assert tree['id_'] == 'root' + assert len(tree['children']) == 1 + # the workflow node - assert tree['type_'] == 'workflow' - assert tree['id_'] == 'workflow id' - assert list(tree['data']) == [ + workflow = tree['children'][0] + assert workflow['type_'] == 'workflow' + assert workflow['id_'] == 'workflow id' + assert set(workflow['data']) == { # whatever if present on the node should end up in data - 'id', 'cyclePoints', 'familyProxies', + 'id', + 'port', 'taskProxies' - ] - assert len(tree['children']) == 1 + } + assert len(workflow['children']) == 1 # the cycle point node - cycle = tree['children'][0] + cycle = workflow['children'][0] assert cycle['type_'] == 'cycle' assert cycle['id_'] == '//1' assert list(cycle['data']) == [