Merge pull request #6440 from hjoliver/cylc-dump-format

Modernise 'cylc dump -t' print format.

changes.d/6440.feat.md

@@ -0,0 +1 @@
+The "cylc dump" command now prints task IDs. Use "--legacy" if you need the old format.

cylc/flow/scripts/dump.py

@@ -162,6 +162,9 @@ def get_option_parser():
     parser.add_option(
         "-t", "--tasks", help="Task states only.",
         action="store_const", const="tasks", dest="disp_form")
+    parser.add_option(
+        "-l", "--legacy", help="Tasks states only; use legacy format.",
+        action="store_true", default=False, dest="legacy_format")
     parser.add_option(
         "-f", "--flows", help="Print flow numbers with tasks.",
         action="store_true", default=False, dest="show_flows")
@@ -274,7 +277,7 @@ async def dump(workflow_id, options, write=print):
                     for key, value in sorted(summary.items()):
                         write(
                             f'{to_snake_case(key).replace("_", " ")}={value}')
-                else:
+                elif options.legacy_format:
                     for item in summary['taskProxies']:
                         if options.sort_by_cycle:
                             values = [
@@ -294,6 +297,27 @@ async def dump(workflow_id, options, write=print):
                         if options.show_flows:
                             values.append(item['flowNums'])
                         write(', '.join(values))
+                else:
+                    for item in summary['taskProxies']:
+                        result = (
+                            f"{item['cyclePoint']}/{item['name']}"
+                            f":{item['state']}"
+                        )
+                        attrs = []
+                        if item['isHeld']:
+                            attrs.append("held")
+                        if item['isQueued']:
+                            attrs.append("queued")
+                        if item['isRunahead']:
+                            attrs.append("runahead")
+                        if attrs:
+                            result += " (" + ",".join(attrs) + ")"
+                        if options.show_flows:
+                            result += (
+                                f" flows={item['flowNums'].replace(' ','')}"
+                            )
+                        write(result)
+
     except Exception as exc:
         raise CylcError(
             json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n'

tests/flakyfunctional/hold-release/14-hold-kill/flow.cylc

@@ -12,7 +12,7 @@
                 '1/sleeper:waiting\(held\).* job killed'
 
             sleep 10  # sleep, should still be held after 10 seconds
-            cylc dump -s -t "${CYLC_WORKFLOW_ID}" >'cylc-dump.out'
+            cylc dump -l -s -t "${CYLC_WORKFLOW_ID}" >'cylc-dump.out'
             diff -u 'cylc-dump.out' - <<'__OUT__'
             1, killer, running, not-held, not-queued, not-runahead
             1, sleeper, waiting, held, not-queued, not-runahead

tests/flakyfunctional/restart/21-task-elapsed.t

@@ -70,7 +70,7 @@ cylc workflow-state "${WORKFLOW_NAME}" \
     --status=running \
     --interval=1 \
     --max-polls=10 1>'/dev/null' 2>&1
-cylc dump -r "${WORKFLOW_NAME}" >'cylc-dump.out'
+cylc dump -l -r "${WORKFLOW_NAME}" >'cylc-dump.out'
 
 test_dump 'cylc-dump.out'
 

tests/functional/queues/qsize/flow.cylc

@@ -16,7 +16,7 @@
         N_SUCCEEDED=0
         while ((N_SUCCEEDED < 12)); do
             sleep 1
-            N_RUNNING=$(cylc dump -t $CYLC_WORKFLOW_ID | grep running | wc -l)
+            N_RUNNING=$(cylc dump -l -t $CYLC_WORKFLOW_ID | grep running | wc -l)
             ((N_RUNNING <= {{q_size}})) # check
             N_SUCCEEDED=$(cylc workflow-state "${CYLC_WORKFLOW_ID}//*/*:succeeded" | wc -l)
         done

tests/functional/reload/03-queues/flow.cylc

@@ -28,7 +28,7 @@ cylc__job__poll_grep_workflow_log 'Reload completed'
         script = """
         cylc__job__wait_cylc_message_started
         while true; do
-            RUNNING=$(cylc dump -t "${CYLC_WORKFLOW_ID}" | grep running | wc -l)
+            RUNNING=$(cylc dump -l -t "${CYLC_WORKFLOW_ID}" | grep running | wc -l)
             # Should be max of: monitor plus 3 members of q1
             echo "RUNNING $RUNNING"
             if ((RUNNING > 4)); then

tests/functional/runahead/06-release-update.t

@@ -34,7 +34,7 @@ poll_grep_workflow_log -E "${NEXT1}/bar.* added to active task pool"
 sleep 10
 
 # (gratuitous use of --flows for test coverage)
-cylc dump --flows -t "${WORKFLOW_NAME}" | awk '{print $1 $2 $3 $7}' >'log'
+cylc dump -l --flows -t "${WORKFLOW_NAME}" | awk '{print $1 $2 $3 $7}' >'log'
 
 # The scheduler task pool should contain:
 #   NEXT1/foo - waiting on clock trigger

tests/functional/spawn-on-demand/13-trigger-runahead/flow.cylc

@@ -14,7 +14,7 @@
          if ((CYLC_TASK_CYCLE_POINT == 1)); then
             expected="foo, 1, running, not-held, not-queued, not-runahead
 foo, 2, waiting, not-held, not-queued, runahead"
-            diff <(cylc dump -t "${CYLC_WORKFLOW_ID}") <(echo "$expected")
+            diff <(cylc dump -l -t "${CYLC_WORKFLOW_ID}") <(echo "$expected")
             # Force trigger next instance while it is runahead limited.
             cylc trigger $CYLC_WORKFLOW_ID//2/foo
          fi

tests/functional/spawn-on-demand/14-trigger-flow-blocker/flow.cylc

@@ -18,7 +18,7 @@
             if ((CYLC_TASK_CYCLE_POINT == 1)); then
                 # Force trigger 3/foo while 2/foo is runahead limited.
                 expected="foo, 2, waiting, not-held, not-queued, runahead"
-                diff <(cylc dump -t "${CYLC_WORKFLOW_ID}" | grep 'foo, 2') \
+                diff <(cylc dump -l -t "${CYLC_WORKFLOW_ID}" | grep 'foo, 2') \
                      <(echo "$expected")
                 cylc trigger --flow=none $CYLC_WORKFLOW_ID//3/foo
             elif ((CYLC_TASK_CYCLE_POINT == 3)); then

tests/integration/scripts/test_dump.py

@@ -16,6 +16,8 @@
 
 """Test the "cylc dump" command."""
 
+import pytest
+
 from cylc.flow.option_parsers import (
     Options,
 )
@@ -24,7 +26,6 @@
     get_option_parser,
 )
 
-
 DumpOptions = Options(get_option_parser())
 
 
@@ -48,5 +49,68 @@ async def test_dump_tasks(flow, scheduler, start):
         # schd.release_queued_tasks()
         await schd.update_data_structure()
         ret = []
-        await dump(id_, DumpOptions(disp_form='tasks'), write=ret.append)
+        await dump(
+            id_,
+            DumpOptions(disp_form='tasks', legacy_format=True),
+            write=ret.append
+        )
         assert ret == ['a, 1, waiting, not-held, queued, not-runahead']
+
+@pytest.mark.parametrize(
+    'attributes_bool, flow_nums, dump_str',
+    [
+        pytest.param(
+            True,
+            [1,2],
+            '1/a:waiting (held,queued,runahead) flows=[1,2]',
+            id='1'
+        ),
+        pytest.param(
+            False,
+            [1,2],
+            '1/a:waiting',
+            id='2'
+        )
+    ]
+ )
+async def test_dump_format(
+    flow, scheduler, start, attributes_bool, flow_nums, dump_str
+):
+    """Check the new "cylc dump" output format, i.e. task IDs.
+
+    See: https://github.com/cylc/cylc-flow/pull/6440
+    """
+    id_ = flow({
+        'scheduler': {
+            'allow implicit tasks': 'true',
+        },
+        'scheduling': {
+            'graph': {
+                'R1': 'a',
+            },
+        },
+    })
+    schd = scheduler(id_)
+    async with start(schd):
+        [itask] = schd.pool.get_tasks()
+
+        itask.state_reset(
+            is_held=attributes_bool,
+            is_runahead=attributes_bool,
+            is_queued=attributes_bool
+        )
+        itask.flow_nums = set(flow_nums)
+
+        schd.pool.data_store_mgr.delta_task_held(
+            itask.tdef.name, itask.point, itask.state.is_held)
+        schd.pool.data_store_mgr.delta_task_state(itask)
+        schd.pool.data_store_mgr.delta_task_flow_nums(itask)
+        await schd.update_data_structure()
+
+        ret = []
+        await dump(
+            id_,
+            DumpOptions(disp_form='tasks', show_flows=attributes_bool),
+            write=ret.append
+        )
+        assert ret == [dump_str]