diff --git a/.codacy.yml b/.codacy.yml index 0c37c0c2898..25f4ed60e5c 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -1,4 +1,4 @@ exclude_paths: - 'etc/**' - 'tests/**' - - 'cylc/flow/**_pb2.py' + - 'cylc/flow/network/protobuf/**_pb2.py' diff --git a/.codecov.yml b/.codecov.yml index 5b3d02564ba..51c036a4457 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -28,7 +28,7 @@ coverage: # files to ignore ignore: - "tests/**" - - "ws_messages_pb2.py" + - "cylc/flow/network/protobuf/cylc/v5/schema_pb2.py" - "cylc/flow/scripts/report_timings.py" flag_management: diff --git a/.coveragerc b/.coveragerc index 5a4b396f19c..3120d48b8a5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -14,7 +14,7 @@ disable_warnings = module-not-measured omit = tests/* - */cylc/flow/*_pb2.py + */cylc/flow/network/protobuf/cylc/v5/*_pb2.py cylc/flow/etc/* cylc/flow/scripts/report_timings.py parallel = True @@ -43,7 +43,7 @@ fail_under=0 ignore_errors = False omit = tests/* - */cylc/flow/*_pb2.py + */cylc/flow/network/protobuf/cylc/v5/*_pb2.py cylc/flow/etc/* precision = 2 show_missing = False diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml new file mode 100644 index 00000000000..e3588a9a2a2 --- /dev/null +++ b/.github/workflows/protobuf.yml @@ -0,0 +1,83 @@ +name: protobuf +# CI tests to run against the protobuf schema on change: + +on: + # run for any PRs raising changes to the protobuf files or setup + pull_request: + paths: + - 'cylc/flow/network/protobuf/**' + +jobs: + protobuf: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we need all of the commits on the PR branch in order to be able to add new ones + fetch-depth: 100 + + - name: Configure git + uses: cylc/release-actions/configure-git@v1 + + - name: Install Protobuf + uses: mamba-org/setup-micromamba@v1 + with: + # install protobuf into a mamba env (note use shell = "bash -el {0}" + # to access this envionment) + environment-name: protobuf + create-args: protobuf + init-shell: bash + + - name: Install bufbuild/buf + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" # activate homebrew + + # NOTE: bufbuild does exist on conda-forge but hasn't been updated for a while + brew install bufbuild/buf/buf + + - name: Lint + run: | + # lint .proto files + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" # activate homebrew + cd cylc/flow/network/protobuf + buf lint + + - name: Compatibility + shell: bash -el {0} + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" # activate homebrew + cd cylc/flow/network/protobuf + # NOTE: there is currently no process for committing a breaking change. + # If a breaking change is needed: + # - Increment the Cylc API version number. + # - Increment the protobuf schema version number to match. + # - Increment the API number filter in cylc-uiserver. + # - Bypass this step of the workflow. + buf breaking \ + --against 'https://github.com/cylc/cylc-flow.git#tag=${{ github.base_ref }},subdir=cylc/flow/network/protobuf' + + - name: Build + shell: bash -el {0} + run: | + # generate .py and .pyi files from the .proto files + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" # activate homebrew + micromamba activate protobuf + cd cylc/flow/network/protobuf + buf generate + + - name: Commit & Push + run: | + if [[ -z $(git diff --stat) ]]; then + echo '::error:: No functional changes made to the protobuf schema' + exit 0 + else + echo '::info:: pushing update commit' + git add -u + git commit -m 'protobuf: updating generated files' + git remote add pr https://github.com/${{ github.event.pull_request.head.repo.owner.login }}/cylc-flow + git push pr HEAD:${{ github.head_ref }} + exit 0 + fi diff --git a/cylc/flow/data_messages_pb2.py b/cylc/flow/data_messages_pb2.py deleted file mode 100644 index 7fb5ae84d24..00000000000 --- a/cylc/flow/data_messages_pb2.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: data_messages.proto -# Protobuf Python Version: 4.25.3 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xd4\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18 \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x12\x1c\n\x0fn_edge_distance\x18( \x01(\x05H\x1c\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updatedB\x12\n\x10_n_edge_distance\"\xe1\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x12\x17\n\ncompletion\x18\x12 \x01(\tH\x11\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputsB\r\n\x0b_completion\"\x9d\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18 \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtimeJ\x04\x08\x1d\x10\x1e\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\x91\x08\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x1d \x01(\x05H\x0f\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xae\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x16 \x01(\x05H\x0f\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'data_messages_pb2', _globals) -if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _globals['_PBWORKFLOW_STATETOTALSENTRY']._options = None - _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_options = b'8\001' - _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._options = None - _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_options = b'8\001' - _globals['_PBTASKPROXY_OUTPUTSENTRY']._options = None - _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_options = b'8\001' - _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._options = None - _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_options = b'8\001' - _globals['_PBTASKPROXY_XTRIGGERSENTRY']._options = None - _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_options = b'8\001' - _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._options = None - _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_options = b'8\001' - _globals['_PBMETA']._serialized_start=24 - _globals['_PBMETA']._serialized_end=174 - _globals['_PBTIMEZONE']._serialized_start=177 - _globals['_PBTIMEZONE']._serialized_end=347 - _globals['_PBTASKPROXYREFS']._serialized_start=349 - _globals['_PBTASKPROXYREFS']._serialized_end=388 - _globals['_PBWORKFLOW']._serialized_start=391 - _globals['_PBWORKFLOW']._serialized_end=2011 - _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_start=1441 - _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_end=1491 - _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_start=1493 - _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_end=1566 - _globals['_PBRUNTIME']._serialized_start=2014 - _globals['_PBRUNTIME']._serialized_end=2879 - _globals['_PBJOB']._serialized_start=2882 - _globals['_PBJOB']._serialized_end=3551 - _globals['_PBTASK']._serialized_start=3554 - _globals['_PBTASK']._serialized_end=3908 - _globals['_PBPOLLTASK']._serialized_start=3911 - _globals['_PBPOLLTASK']._serialized_end=4127 - _globals['_PBCONDITION']._serialized_start=4130 - _globals['_PBCONDITION']._serialized_end=4333 - _globals['_PBPREREQUISITE']._serialized_start=4336 - _globals['_PBPREREQUISITE']._serialized_end=4486 - _globals['_PBOUTPUT']._serialized_start=4489 - _globals['_PBOUTPUT']._serialized_end=4629 - _globals['_PBTRIGGER']._serialized_start=4632 - _globals['_PBTRIGGER']._serialized_end=4797 - _globals['_PBTASKPROXY']._serialized_start=4800 - _globals['_PBTASKPROXY']._serialized_end=5841 - _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=5451 - _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=5508 - _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=5510 - _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=5577 - _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=5579 - _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=5639 - _globals['_PBFAMILY']._serialized_start=5844 - _globals['_PBFAMILY']._serialized_end=6172 - _globals['_PBFAMILYPROXY']._serialized_start=6175 - _globals['_PBFAMILYPROXY']._serialized_end=6989 - _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_start=1441 - _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_end=1491 - _globals['_PBEDGE']._serialized_start=6992 - _globals['_PBEDGE']._serialized_end=7180 - _globals['_PBEDGES']._serialized_start=7182 - _globals['_PBEDGES']._serialized_end=7305 - _globals['_PBENTIREWORKFLOW']._serialized_start=7308 - _globals['_PBENTIREWORKFLOW']._serialized_end=7550 - _globals['_EDELTAS']._serialized_start=7553 - _globals['_EDELTAS']._serialized_end=7728 - _globals['_FDELTAS']._serialized_start=7731 - _globals['_FDELTAS']._serialized_end=7910 - _globals['_FPDELTAS']._serialized_start=7913 - _globals['_FPDELTAS']._serialized_end=8103 - _globals['_JDELTAS']._serialized_start=8106 - _globals['_JDELTAS']._serialized_end=8279 - _globals['_TDELTAS']._serialized_start=8282 - _globals['_TDELTAS']._serialized_end=8457 - _globals['_TPDELTAS']._serialized_start=8460 - _globals['_TPDELTAS']._serialized_end=8646 - _globals['_WDELTAS']._serialized_start=8649 - _globals['_WDELTAS']._serialized_end=8844 - _globals['_ALLDELTAS']._serialized_start=8847 - _globals['_ALLDELTAS']._serialized_end=9056 -# @@protoc_insertion_point(module_scope) diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py index c59fa7b6c62..eec3f7c89cd 100644 --- a/cylc/flow/data_store_mgr.py +++ b/cylc/flow/data_store_mgr.py @@ -73,10 +73,25 @@ from cylc.flow import __version__ as CYLC_VERSION, LOG from cylc.flow.cycling.loader import get_point -from cylc.flow.data_messages_pb2 import ( - PbEdge, PbEntireWorkflow, PbFamily, PbFamilyProxy, PbJob, PbTask, - PbTaskProxy, PbWorkflow, PbRuntime, AllDeltas, EDeltas, FDeltas, - FPDeltas, JDeltas, TDeltas, TPDeltas, WDeltas) +from cylc.flow.network.protobuf.cylc.v5.schema_pb2 import ( + PbEdge, + PbEntireWorkflow, + PbFamily, + PbFamilyProxy, + PbJob, + PbTask, + PbTaskProxy, + PbWorkflow, + PbRuntime, + AllDeltas, + EDeltas, + FDeltas, + FPDeltas, + JDeltas, + TDeltas, + TPDeltas, + WDeltas, +) from cylc.flow.exceptions import WorkflowConfigError from cylc.flow.id import Tokens from cylc.flow.network import API @@ -382,7 +397,7 @@ def create_delta_store(delta=None, workflow_id=None): """Create a mini data-store out of the all deltas message. Args: - delta (cylc.flow.data_messages_pb2.AllDeltas): + delta (AllDeltas): The message of accumulated deltas for publish/push. workflow_id (str): The workflow ID. @@ -430,18 +445,18 @@ class DataStoreMgr: Local store of config.get_first_parent_ancestors() .data (dict): .edges (dict): - cylc.flow.data_messages_pb2.PbEdge by internal ID. + PbEdge by internal ID. .families (dict): - cylc.flow.data_messages_pb2.PbFamily by name (internal ID). + PbFamily by name (internal ID). .family_proxies (dict): - cylc.flow.data_messages_pb2.PbFamilyProxy by internal ID. + PbFamilyProxy by internal ID. .jobs (dict): - cylc.flow.data_messages_pb2.PbJob by internal ID. + PbJob by internal ID. .tasks (dict): - cylc.flow.data_messages_pb2.PbTask by name (internal ID). + PbTask by name (internal ID). .task_proxies (dict): - cylc.flow.data_messages_pb2.PbTaskProxy by internal ID. - .workflow (cylc.flow.data_messages_pb2.PbWorkflow) + PbTaskProxy by internal ID. + .workflow (PbWorkflow) Message containing the global information of the workflow. .descendants (dict): Local store of config.get_first_parent_descendants() @@ -2688,7 +2703,7 @@ def get_entire_workflow(self): """Gather data elements into single Protobuf message. Returns: - cylc.flow.data_messages_pb2.PbEntireWorkflow + PbEntireWorkflow """ diff --git a/cylc/flow/network/__init__.py b/cylc/flow/network/__init__.py index 916b129e244..96da617db40 100644 --- a/cylc/flow/network/__init__.py +++ b/cylc/flow/network/__init__.py @@ -13,279 +13,18 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Package for network interfaces to Cylc scheduler objects.""" -import asyncio -import getpass -import json -from typing import Optional, Tuple +"""Cylc networking code. -import zmq -import zmq.asyncio -import zmq.auth +Contains: +* Server code (hosted by the scheduler process). +* Client implementations (used to communicate with the scheduler). +* Workflow scanning logic. +* Schema and interface definitions. +""" -from cylc.flow import LOG -from cylc.flow.exceptions import ( - ClientError, - CylcError, - CylcVersionError, - ServiceFileError, - WorkflowStopped -) -from cylc.flow.hostuserutil import get_fqdn_by_host -from cylc.flow.workflow_files import ( - ContactFileFields, - KeyType, - KeyOwner, - KeyInfo, - load_contact_file, - get_workflow_srv_dir -) - -API = 5 # cylc API version -MSG_TIMEOUT = "TIMEOUT" - - -def encode_(message): - """Convert the structure holding a message field from JSON to a string.""" - try: - return json.dumps(message) - except TypeError as exc: - return json.dumps({'errors': [{'message': str(exc)}]}) - - -def decode_(message): - """Convert an encoded message string to JSON with an added 'user' field.""" - msg = json.loads(message) - msg['user'] = getpass.getuser() # assume this is the user - return msg - - -def get_location(workflow: str) -> Tuple[str, int, int]: - """Extract host and port from a workflow's contact file. - - NB: if it fails to load the workflow contact file, it will exit. - - Args: - workflow: workflow ID - Returns: - Tuple (host name, port number, publish port number) - Raises: - WorkflowStopped: if the workflow is not running. - CylcVersionError: if target is a Cylc 7 (or earlier) workflow. - """ - try: - contact = load_contact_file(workflow) - except (IOError, ValueError, ServiceFileError): - # Contact file does not exist or corrupted, workflow should be dead - raise WorkflowStopped(workflow) - - host = contact[ContactFileFields.HOST] - host = get_fqdn_by_host(host) - port = int(contact[ContactFileFields.PORT]) - if ContactFileFields.PUBLISH_PORT in contact: - pub_port = int(contact[ContactFileFields.PUBLISH_PORT]) - else: - version = contact.get('CYLC_VERSION', None) - raise CylcVersionError(version=version) - return host, port, pub_port - - -class ZMQSocketBase: - """Initiate the ZMQ socket bind for specified pattern. - - NOTE: Security to be provided via zmq.auth (see PR #3359). - - Args: - pattern (enum): ZeroMQ message pattern (zmq.PATTERN). - - context (object, optional): instantiated ZeroMQ context, defaults - to zmq.asyncio.Context(). - - This class is designed to be inherited by REP Server (REQ/REP) - and by PUB Publisher (PUB/SUB), as the start-up logic is similar. - - - To tailor this class overwrite it's method on inheritance. - - """ - - def __init__( - self, - pattern, - workflow: str, - bind: bool = False, - context: Optional[zmq.Context] = None, - ): - self.bind = bind - if context is None: - self.context: zmq.Context = zmq.asyncio.Context() - else: - self.context = context - self.pattern = pattern - self.workflow = workflow - self.host: Optional[str] = None - self.port: Optional[int] = None - self.socket: Optional[zmq.Socket] = None - self.loop: Optional[asyncio.AbstractEventLoop] = None - self.stopping = False - - def start(self, *args, **kwargs): - """Create the async loop, and bind socket.""" - # set asyncio loop - try: - self.loop = asyncio.get_running_loop() - except RuntimeError: - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - - if self.bind: - self._socket_bind(*args, **kwargs) - else: - self._socket_connect(*args, **kwargs) - - # initiate bespoke items - self._bespoke_start() - - # Keeping srv_prv_key_loc as optional arg so as to not break interface - def _socket_bind(self, min_port, max_port, srv_prv_key_loc=None): - """Bind socket. - - Will use a port range provided to select random ports. - - """ - if srv_prv_key_loc is None: - # Create new KeyInfo object for the server private key - workflow_srv_dir = get_workflow_srv_dir(self.workflow) - srv_prv_key_info = KeyInfo( - KeyType.PRIVATE, - KeyOwner.SERVER, - workflow_srv_dir=workflow_srv_dir) - else: - srv_prv_key_info = KeyInfo( - KeyType.PRIVATE, - KeyOwner.SERVER, - full_key_path=srv_prv_key_loc) - - # create socket - self.socket = self.context.socket(self.pattern) - self._socket_options() - - try: - server_public_key, server_private_key = zmq.auth.load_certificate( - srv_prv_key_info.full_key_path) - except ValueError: - raise ServiceFileError( - f"Failed to find server's public " - f"key in " - f"{srv_prv_key_info.full_key_path}." - ) - except OSError: - raise ServiceFileError( - f"IO error opening server's private " - f"key from " - f"{srv_prv_key_info.full_key_path}." - ) - if server_private_key is None: # this can't be caught by exception - raise ServiceFileError( - f"Failed to find server's private " - f"key in " - f"{srv_prv_key_info.full_key_path}." - ) - self.socket.curve_publickey = server_public_key - self.socket.curve_secretkey = server_private_key - self.socket.curve_server = True - - try: - if min_port == max_port: - self.port = min_port - self.socket.bind(f'tcp://*:{min_port}') - else: - self.port = self.socket.bind_to_random_port( - 'tcp://*', min_port, max_port) - except (zmq.error.ZMQError, zmq.error.ZMQBindError) as exc: - raise CylcError(f'could not start Cylc ZMQ server: {exc}') - - # Keeping srv_public_key_loc as optional arg so as to not break interface - def _socket_connect(self, host, port, srv_public_key_loc=None): - """Connect socket to stub.""" - workflow_srv_dir = get_workflow_srv_dir(self.workflow) - if srv_public_key_loc is None: - # Create new KeyInfo object for the server public key - srv_pub_key_info = KeyInfo( - KeyType.PUBLIC, - KeyOwner.SERVER, - workflow_srv_dir=workflow_srv_dir) - - else: - srv_pub_key_info = KeyInfo( - KeyType.PUBLIC, - KeyOwner.SERVER, - full_key_path=srv_public_key_loc) - - self.host = host - self.port = port - self.socket = self.context.socket(self.pattern) - self._socket_options() - - client_priv_key_info = KeyInfo( - KeyType.PRIVATE, - KeyOwner.CLIENT, - workflow_srv_dir=workflow_srv_dir) - error_msg = "Failed to find user's private key, so cannot connect." - try: - client_public_key, client_priv_key = zmq.auth.load_certificate( - client_priv_key_info.full_key_path) - except (OSError, ValueError): - raise ClientError(error_msg) - if client_priv_key is None: # this can't be caught by exception - raise ClientError(error_msg) - self.socket.curve_publickey = client_public_key - self.socket.curve_secretkey = client_priv_key - - # A client can only connect to the server if it knows its public key, - # so we grab this from the location it was created on the filesystem: - try: - # 'load_certificate' will try to load both public & private keys - # from a provided file but will return None, not throw an error, - # for the latter item if not there (as for all public key files) - # so it is OK to use; there is no method to load only the - # public key. - server_public_key = zmq.auth.load_certificate( - srv_pub_key_info.full_key_path)[0] - self.socket.curve_serverkey = server_public_key - except (OSError, ValueError): # ValueError raised w/ no public key - raise ClientError( - "Failed to load the workflow's public key, so cannot connect.") - - self.socket.connect(f'tcp://{host}:{port}') - - def _socket_options(self): - """Set socket options. - - i.e. self.socket.sndhwm - """ - self.socket.sndhwm = 10000 - - def _bespoke_start(self): - """Initiate bespoke items at start.""" - self.stopping = False - - def stop(self, stop_loop=True): - """Stop the server. - - Args: - stop_loop (Boolean): Stop running IOLoop. - - """ - self._bespoke_stop() - if stop_loop and self.loop and self.loop.is_running(): - self.loop.stop() - if self.socket and not self.socket.closed: - self.socket.close() - LOG.debug('...stopped') - - def _bespoke_stop(self): - """Bespoke stop items.""" - LOG.debug('stopping zmq socket...') - self.stopping = True +# Cylc API version. +# This is the Cylc protocol version number that determines whether a client can +# communicate with a server. This should be changed when breaking changes are +# made for which backwards compatibility can not be provided. +API = 5 diff --git a/cylc/flow/network/base.py b/cylc/flow/network/base.py new file mode 100644 index 00000000000..1842407b448 --- /dev/null +++ b/cylc/flow/network/base.py @@ -0,0 +1,237 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Base ZMQ socket implementation for network server/client implementations.""" + +import asyncio +from typing import Optional + +import zmq +import zmq.asyncio +import zmq.auth + +from cylc.flow import LOG +from cylc.flow.exceptions import ( + ClientError, + CylcError, + ServiceFileError, +) +from cylc.flow.workflow_files import ( + KeyType, + KeyOwner, + KeyInfo, + get_workflow_srv_dir, +) + + +class ZMQSocketBase: + """Initiate the ZMQ socket bind for specified pattern. + + NOTE: Security to be provided via zmq.auth (see PR #3359). + + Args: + pattern (enum): ZeroMQ message pattern (zmq.PATTERN). + + context (object, optional): instantiated ZeroMQ context, defaults + to zmq.asyncio.Context(). + + This class is designed to be inherited by REP Server (REQ/REP) + and by PUB Publisher (PUB/SUB), as the start-up logic is similar. + + + To tailor this class overwrite it's method on inheritance. + + """ + + def __init__( + self, + pattern, + workflow: str, + bind: bool = False, + context: Optional[zmq.Context] = None, + ): + self.bind = bind + if context is None: + self.context: zmq.Context = zmq.asyncio.Context() + else: + self.context = context + self.pattern = pattern + self.workflow = workflow + self.host: Optional[str] = None + self.port: Optional[int] = None + self.socket: Optional[zmq.Socket] = None + self.loop: Optional[asyncio.AbstractEventLoop] = None + self.stopping = False + + def start(self, *args, **kwargs): + """Create the async loop, and bind socket.""" + # set asyncio loop + try: + self.loop = asyncio.get_running_loop() + except RuntimeError: + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + if self.bind: + self._socket_bind(*args, **kwargs) + else: + self._socket_connect(*args, **kwargs) + + # initiate bespoke items + self._bespoke_start() + + # Keeping srv_prv_key_loc as optional arg so as to not break interface + def _socket_bind(self, min_port, max_port, srv_prv_key_loc=None): + """Bind socket. + + Will use a port range provided to select random ports. + + """ + if srv_prv_key_loc is None: + # Create new KeyInfo object for the server private key + workflow_srv_dir = get_workflow_srv_dir(self.workflow) + srv_prv_key_info = KeyInfo( + KeyType.PRIVATE, + KeyOwner.SERVER, + workflow_srv_dir=workflow_srv_dir) + else: + srv_prv_key_info = KeyInfo( + KeyType.PRIVATE, + KeyOwner.SERVER, + full_key_path=srv_prv_key_loc) + + # create socket + self.socket = self.context.socket(self.pattern) + self._socket_options() + + try: + server_public_key, server_private_key = zmq.auth.load_certificate( + srv_prv_key_info.full_key_path) + except ValueError: + raise ServiceFileError( + f"Failed to find server's public " + f"key in " + f"{srv_prv_key_info.full_key_path}." + ) + except OSError: + raise ServiceFileError( + f"IO error opening server's private " + f"key from " + f"{srv_prv_key_info.full_key_path}." + ) + if server_private_key is None: # this can't be caught by exception + raise ServiceFileError( + f"Failed to find server's private " + f"key in " + f"{srv_prv_key_info.full_key_path}." + ) + self.socket.curve_publickey = server_public_key + self.socket.curve_secretkey = server_private_key + self.socket.curve_server = True + + try: + if min_port == max_port: + self.port = min_port + self.socket.bind(f'tcp://*:{min_port}') + else: + self.port = self.socket.bind_to_random_port( + 'tcp://*', min_port, max_port) + except (zmq.error.ZMQError, zmq.error.ZMQBindError) as exc: + raise CylcError(f'could not start Cylc ZMQ server: {exc}') + + # Keeping srv_public_key_loc as optional arg so as to not break interface + def _socket_connect(self, host, port, srv_public_key_loc=None): + """Connect socket to stub.""" + workflow_srv_dir = get_workflow_srv_dir(self.workflow) + if srv_public_key_loc is None: + # Create new KeyInfo object for the server public key + srv_pub_key_info = KeyInfo( + KeyType.PUBLIC, + KeyOwner.SERVER, + workflow_srv_dir=workflow_srv_dir) + + else: + srv_pub_key_info = KeyInfo( + KeyType.PUBLIC, + KeyOwner.SERVER, + full_key_path=srv_public_key_loc) + + self.host = host + self.port = port + self.socket = self.context.socket(self.pattern) + self._socket_options() + + client_priv_key_info = KeyInfo( + KeyType.PRIVATE, + KeyOwner.CLIENT, + workflow_srv_dir=workflow_srv_dir) + error_msg = "Failed to find user's private key, so cannot connect." + try: + client_public_key, client_priv_key = zmq.auth.load_certificate( + client_priv_key_info.full_key_path) + except (OSError, ValueError): + raise ClientError(error_msg) + if client_priv_key is None: # this can't be caught by exception + raise ClientError(error_msg) + self.socket.curve_publickey = client_public_key + self.socket.curve_secretkey = client_priv_key + + # A client can only connect to the server if it knows its public key, + # so we grab this from the location it was created on the filesystem: + try: + # 'load_certificate' will try to load both public & private keys + # from a provided file but will return None, not throw an error, + # for the latter item if not there (as for all public key files) + # so it is OK to use; there is no method to load only the + # public key. + server_public_key = zmq.auth.load_certificate( + srv_pub_key_info.full_key_path)[0] + self.socket.curve_serverkey = server_public_key + except (OSError, ValueError): # ValueError raised w/ no public key + raise ClientError( + "Failed to load the workflow's public key, so cannot connect.") + + self.socket.connect(f'tcp://{host}:{port}') + + def _socket_options(self): + """Set socket options. + + i.e. self.socket.sndhwm + """ + self.socket.sndhwm = 10000 + + def _bespoke_start(self): + """Initiate bespoke items at start.""" + self.stopping = False + + def stop(self, stop_loop=True): + """Stop the server. + + Args: + stop_loop (Boolean): Stop running IOLoop. + + """ + self._bespoke_stop() + if stop_loop and self.loop and self.loop.is_running(): + self.loop.stop() + if self.socket and not self.socket.closed: + self.socket.close() + LOG.debug('...stopped') + + def _bespoke_stop(self): + """Bespoke stop items.""" + LOG.debug('stopping zmq socket...') + self.stopping = True diff --git a/cylc/flow/network/client.py b/cylc/flow/network/client.py index e7e26954d56..6f8206ee786 100644 --- a/cylc/flow/network/client.py +++ b/cylc/flow/network/client.py @@ -35,14 +35,14 @@ WorkflowStopped, ) from cylc.flow.hostuserutil import get_fqdn_by_host -from cylc.flow.network import ( +from cylc.flow.network.base import ZMQSocketBase +from cylc.flow.network.client_factory import CommsMeth +from cylc.flow.network.server import PB_METHOD_MAP +from cylc.flow.network.util import ( encode_, decode_, get_location, - ZMQSocketBase ) -from cylc.flow.network.client_factory import CommsMeth -from cylc.flow.network.server import PB_METHOD_MAP from cylc.flow.workflow_files import ( detect_old_contact_file, ) diff --git a/cylc/flow/network/protobuf/buf.gen.yaml b/cylc/flow/network/protobuf/buf.gen.yaml new file mode 100644 index 00000000000..ad4f94328d9 --- /dev/null +++ b/cylc/flow/network/protobuf/buf.gen.yaml @@ -0,0 +1,10 @@ +# file containing protobuf configuration + +version: v2 + +plugins: + - protoc_builtin: python + out: . + + - protoc_builtin: pyi + out: . diff --git a/cylc/flow/network/protobuf/buf.yaml b/cylc/flow/network/protobuf/buf.yaml new file mode 100644 index 00000000000..27a2c58ce65 --- /dev/null +++ b/cylc/flow/network/protobuf/buf.yaml @@ -0,0 +1,11 @@ +# file containing protobuf configuration + +version: v2 + +lint: + use: + - DEFAULT + +breaking: + use: + - FILE diff --git a/cylc/flow/data_messages.proto b/cylc/flow/network/protobuf/cylc/v5/schema.proto similarity index 94% rename from cylc/flow/data_messages.proto rename to cylc/flow/network/protobuf/cylc/v5/schema.proto index c0af5094c0d..b43bd1aaa27 100644 --- a/cylc/flow/data_messages.proto +++ b/cylc/flow/network/protobuf/cylc/v5/schema.proto @@ -1,5 +1,7 @@ syntax = "proto3"; +package cylc.v5; + /* THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. Copyright (C) NIWA & British Crown (Met Office) & Contributors. @@ -16,6 +18,7 @@ syntax = "proto3"; You should have received a copy of the GNU General Public License along with this program. If not, see .*/ + /* Protobuf message definitions * * The original intention of these messages is for use as data elements sent @@ -24,15 +27,21 @@ syntax = "proto3"; * This file is not needed at runtime. It is used to generate python protobuf * message modules. * - * Command: - * $ protoc -I=./ --python_out=./ --pyi_out=./ data_messages.proto + * Hand edit this file to make changes to the schema, use "bufbuild" to lint + * changes and regenerate Python files e.g: * - * Pre-compiled protoc binary may be download from: - * https://github.com/protocolbuffers/protobuf/releases + * # cd into the directory which contains the "buf.yaml" file, this is the + * # project's root directory as far as protobuf is concerned + * $ cd cylc/flow/network/protobuf * - * If merge/rebase conflicts arise, then regenerate the module. - * (DO NOT manually resolve conflicts) + * # install bufbuild + * $ npm install @bufbuild/buf * + * # lint protobuf schema source files + * node_modules/@bufbuild/buf/bin/buf lint + * + * # generate Python files + * node_modules/@bufbuild/buf/bin/buf generate * * WARNING: Avoid re-indexing existing fields! * - Field numbers do not need to be continuous/sequential (gaps are fine). @@ -43,7 +52,6 @@ syntax = "proto3"; * * https://developers.google.com/protocol-buffers/docs/proto3#assigning_field_numbers * - * * */ @@ -51,6 +59,7 @@ syntax = "proto3"; message PbMeta { optional string title = 1; optional string description = 2; + // buf:lint:ignore FIELD_LOWER_SNAKE_CASE optional string URL = 3; optional string user_defined = 4; } diff --git a/cylc/flow/network/protobuf/cylc/v5/schema_pb2.py b/cylc/flow/network/protobuf/cylc/v5/schema_pb2.py new file mode 100644 index 00000000000..4de0f36dda6 --- /dev/null +++ b/cylc/flow/network/protobuf/cylc/v5/schema_pb2.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: cylc/v5/schema.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14\x63ylc/v5/schema.proto\x12\x07\x63ylc.v5\"\xbc\x01\n\x06PbMeta\x12\x19\n\x05title\x18\x01 \x01(\tH\x00R\x05title\x88\x01\x01\x12%\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01R\x0b\x64\x65scription\x88\x01\x01\x12\x15\n\x03URL\x18\x03 \x01(\tH\x02R\x03URL\x88\x01\x01\x12&\n\x0cuser_defined\x18\x04 \x01(\tH\x03R\x0buserDefined\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xd7\x01\n\nPbTimeZone\x12\x19\n\x05hours\x18\x01 \x01(\x05H\x00R\x05hours\x88\x01\x01\x12\x1d\n\x07minutes\x18\x02 \x01(\x05H\x01R\x07minutes\x88\x01\x01\x12&\n\x0cstring_basic\x18\x03 \x01(\tH\x02R\x0bstringBasic\x88\x01\x01\x12,\n\x0fstring_extended\x18\x04 \x01(\tH\x03R\x0estringExtended\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"4\n\x0fPbTaskProxyRefs\x12!\n\x0ctask_proxies\x18\x01 \x03(\tR\x0btaskProxies\"\xda\x10\n\nPbWorkflow\x12\x19\n\x05stamp\x18\x01 \x01(\tH\x00R\x05stamp\x88\x01\x01\x12\x13\n\x02id\x18\x02 \x01(\tH\x01R\x02id\x88\x01\x01\x12\x17\n\x04name\x18\x03 \x01(\tH\x02R\x04name\x88\x01\x01\x12\x1b\n\x06status\x18\x04 \x01(\tH\x03R\x06status\x88\x01\x01\x12\x17\n\x04host\x18\x05 \x01(\tH\x04R\x04host\x88\x01\x01\x12\x17\n\x04port\x18\x06 \x01(\x05H\x05R\x04port\x88\x01\x01\x12\x19\n\x05owner\x18\x07 \x01(\tH\x06R\x05owner\x88\x01\x01\x12\x14\n\x05tasks\x18\x08 \x03(\tR\x05tasks\x12\x1a\n\x08\x66\x61milies\x18\t \x03(\tR\x08\x66\x61milies\x12+\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x10.cylc.v5.PbEdgesH\x07R\x05\x65\x64ges\x88\x01\x01\x12$\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08R\napiVersion\x88\x01\x01\x12&\n\x0c\x63ylc_version\x18\x0c \x01(\tH\tR\x0b\x63ylcVersion\x88\x01\x01\x12&\n\x0clast_updated\x18\r \x01(\x01H\nR\x0blastUpdated\x88\x01\x01\x12(\n\x04meta\x18\x0e \x01(\x0b\x32\x0f.cylc.v5.PbMetaH\x0bR\x04meta\x88\x01\x01\x12>\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0cR\x16newestActiveCyclePoint\x88\x01\x01\x12>\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\rR\x16oldestActiveCyclePoint\x88\x01\x01\x12\x1f\n\x08reloaded\x18\x12 \x01(\x08H\x0eR\x08reloaded\x88\x01\x01\x12\x1e\n\x08run_mode\x18\x13 \x01(\tH\x0fR\x07runMode\x88\x01\x01\x12&\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10R\x0b\x63yclingMode\x88\x01\x01\x12G\n\x0cstate_totals\x18\x15 \x03(\x0b\x32$.cylc.v5.PbWorkflow.StateTotalsEntryR\x0bstateTotals\x12-\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11R\x0eworkflowLogDir\x88\x01\x01\x12>\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x13.cylc.v5.PbTimeZoneH\x12R\x0ctimeZoneInfo\x88\x01\x01\x12\"\n\ntree_depth\x18\x18 \x01(\x05H\x13R\ttreeDepth\x88\x01\x01\x12\"\n\rjob_log_names\x18\x19 \x03(\tR\x0bjobLogNames\x12 \n\x0cns_def_order\x18\x1a \x03(\tR\nnsDefOrder\x12\x16\n\x06states\x18\x1b \x03(\tR\x06states\x12!\n\x0ctask_proxies\x18\x1c \x03(\tR\x0btaskProxies\x12%\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\tR\rfamilyProxies\x12\"\n\nstatus_msg\x18\x1e \x01(\tH\x14R\tstatusMsg\x88\x01\x01\x12\'\n\ris_held_total\x18\x1f \x01(\x05H\x15R\x0bisHeldTotal\x88\x01\x01\x12\x12\n\x04jobs\x18 \x03(\tR\x04jobs\x12\x1e\n\x08pub_port\x18! \x01(\x05H\x16R\x07pubPort\x88\x01\x01\x12#\n\nbroadcasts\x18\" \x01(\tH\x17R\nbroadcasts\x88\x01\x01\x12+\n\x0fis_queued_total\x18# \x01(\x05H\x18R\risQueuedTotal\x88\x01\x01\x12W\n\x12latest_state_tasks\x18$ \x03(\x0b\x32).cylc.v5.PbWorkflow.LatestStateTasksEntryR\x10latestStateTasks\x12\x1b\n\x06pruned\x18% \x01(\x08H\x19R\x06pruned\x88\x01\x01\x12/\n\x11is_runahead_total\x18& \x01(\x05H\x1aR\x0fisRunaheadTotal\x88\x01\x01\x12*\n\x0estates_updated\x18\' \x01(\x08H\x1bR\rstatesUpdated\x88\x01\x01\x12+\n\x0fn_edge_distance\x18( \x01(\x05H\x1cR\rnEdgeDistance\x88\x01\x01\x1a>\n\x10StateTotalsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\x05R\x05value:\x02\x38\x01\x1a]\n\x15LatestStateTasksEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x18.cylc.v5.PbTaskProxyRefsR\x05value:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updatedB\x12\n\x10_n_edge_distance\"\xea\x08\n\tPbRuntime\x12\x1f\n\x08platform\x18\x01 \x01(\tH\x00R\x08platform\x88\x01\x01\x12\x1b\n\x06script\x18\x02 \x01(\tH\x01R\x06script\x88\x01\x01\x12$\n\x0binit_script\x18\x03 \x01(\tH\x02R\ninitScript\x88\x01\x01\x12\"\n\nenv_script\x18\x04 \x01(\tH\x03R\tenvScript\x88\x01\x01\x12\"\n\nerr_script\x18\x05 \x01(\tH\x04R\terrScript\x88\x01\x01\x12$\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05R\nexitScript\x88\x01\x01\x12\"\n\npre_script\x18\x07 \x01(\tH\x06R\tpreScript\x88\x01\x01\x12$\n\x0bpost_script\x18\x08 \x01(\tH\x07R\npostScript\x88\x01\x01\x12%\n\x0cwork_sub_dir\x18\t \x01(\tH\x08R\nworkSubDir\x88\x01\x01\x12\x43\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\tR\x19\x65xecutionPollingIntervals\x88\x01\x01\x12\x39\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\nR\x14\x65xecutionRetryDelays\x88\x01\x01\x12\x35\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0bR\x12\x65xecutionTimeLimit\x88\x01\x01\x12\x45\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0cR\x1asubmissionPollingIntervals\x88\x01\x01\x12;\n\x17submission_retry_delays\x18\x0e \x01(\tH\rR\x15submissionRetryDelays\x88\x01\x01\x12#\n\ndirectives\x18\x0f \x01(\tH\x0eR\ndirectives\x88\x01\x01\x12%\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0fR\x0b\x65nvironment\x88\x01\x01\x12\x1d\n\x07outputs\x18\x11 \x01(\tH\x10R\x07outputs\x88\x01\x01\x12#\n\ncompletion\x18\x12 \x01(\tH\x11R\ncompletion\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputsB\r\n\x0b_completion\"\xdb\x06\n\x05PbJob\x12\x19\n\x05stamp\x18\x01 \x01(\tH\x00R\x05stamp\x88\x01\x01\x12\x13\n\x02id\x18\x02 \x01(\tH\x01R\x02id\x88\x01\x01\x12\"\n\nsubmit_num\x18\x03 \x01(\x05H\x02R\tsubmitNum\x88\x01\x01\x12\x19\n\x05state\x18\x04 \x01(\tH\x03R\x05state\x88\x01\x01\x12\"\n\ntask_proxy\x18\x05 \x01(\tH\x04R\ttaskProxy\x88\x01\x01\x12*\n\x0esubmitted_time\x18\x06 \x01(\tH\x05R\rsubmittedTime\x88\x01\x01\x12&\n\x0cstarted_time\x18\x07 \x01(\tH\x06R\x0bstartedTime\x88\x01\x01\x12(\n\rfinished_time\x18\x08 \x01(\tH\x07R\x0c\x66inishedTime\x88\x01\x01\x12\x1a\n\x06job_id\x18\t \x01(\tH\x08R\x05jobId\x88\x01\x01\x12+\n\x0fjob_runner_name\x18\n \x01(\tH\tR\rjobRunnerName\x88\x01\x01\x12\x35\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\nR\x12\x65xecutionTimeLimit\x88\x01\x01\x12\x1f\n\x08platform\x18\x0f \x01(\tH\x0bR\x08platform\x88\x01\x01\x12#\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0cR\tjobLogDir\x88\x01\x01\x12\x17\n\x04name\x18\x1e \x01(\tH\rR\x04name\x88\x01\x01\x12$\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0eR\ncyclePoint\x88\x01\x01\x12\x1a\n\x08messages\x18 \x03(\tR\x08messages\x12\x31\n\x07runtime\x18! \x01(\x0b\x32\x12.cylc.v5.PbRuntimeH\x0fR\x07runtime\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtimeJ\x04\x08\x1d\x10\x1e\"\xd4\x03\n\x06PbTask\x12\x19\n\x05stamp\x18\x01 \x01(\tH\x00R\x05stamp\x88\x01\x01\x12\x13\n\x02id\x18\x02 \x01(\tH\x01R\x02id\x88\x01\x01\x12\x17\n\x04name\x18\x03 \x01(\tH\x02R\x04name\x88\x01\x01\x12(\n\x04meta\x18\x04 \x01(\x0b\x32\x0f.cylc.v5.PbMetaH\x03R\x04meta\x88\x01\x01\x12/\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04R\x0fmeanElapsedTime\x88\x01\x01\x12\x19\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05R\x05\x64\x65pth\x88\x01\x01\x12\x18\n\x07proxies\x18\x07 \x03(\tR\x07proxies\x12\x1c\n\tnamespace\x18\x08 \x03(\tR\tnamespace\x12\x18\n\x07parents\x18\t \x03(\tR\x07parents\x12&\n\x0c\x66irst_parent\x18\n \x01(\tH\x06R\x0b\x66irstParent\x88\x01\x01\x12\x31\n\x07runtime\x18\x0b \x01(\x0b\x32\x12.cylc.v5.PbRuntimeH\x07R\x07runtime\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\x92\x02\n\nPbPollTask\x12$\n\x0blocal_proxy\x18\x01 \x01(\tH\x00R\nlocalProxy\x88\x01\x01\x12\x1f\n\x08workflow\x18\x02 \x01(\tH\x01R\x08workflow\x88\x01\x01\x12&\n\x0cremote_proxy\x18\x03 \x01(\tH\x02R\x0bremoteProxy\x88\x01\x01\x12 \n\treq_state\x18\x04 \x01(\tH\x03R\x08reqState\x88\x01\x01\x12&\n\x0cgraph_string\x18\x05 \x01(\tH\x04R\x0bgraphString\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xff\x01\n\x0bPbCondition\x12\"\n\ntask_proxy\x18\x01 \x01(\tH\x00R\ttaskProxy\x88\x01\x01\x12\"\n\nexpr_alias\x18\x02 \x01(\tH\x01R\texprAlias\x88\x01\x01\x12 \n\treq_state\x18\x03 \x01(\tH\x02R\x08reqState\x88\x01\x01\x12!\n\tsatisfied\x18\x04 \x01(\x08H\x03R\tsatisfied\x88\x01\x01\x12\x1d\n\x07message\x18\x05 \x01(\tH\x04R\x07message\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\xce\x01\n\x0ePbPrerequisite\x12#\n\nexpression\x18\x01 \x01(\tH\x00R\nexpression\x88\x01\x01\x12\x34\n\nconditions\x18\x02 \x03(\x0b\x32\x14.cylc.v5.PbConditionR\nconditions\x12!\n\x0c\x63ycle_points\x18\x03 \x03(\tR\x0b\x63yclePoints\x12!\n\tsatisfied\x18\x04 \x01(\x08H\x01R\tsatisfied\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\xad\x01\n\x08PbOutput\x12\x19\n\x05label\x18\x01 \x01(\tH\x00R\x05label\x88\x01\x01\x12\x1d\n\x07message\x18\x02 \x01(\tH\x01R\x07message\x88\x01\x01\x12!\n\tsatisfied\x18\x03 \x01(\x08H\x02R\tsatisfied\x88\x01\x01\x12\x17\n\x04time\x18\x04 \x01(\x01H\x03R\x04time\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xca\x01\n\tPbTrigger\x12\x13\n\x02id\x18\x01 \x01(\tH\x00R\x02id\x88\x01\x01\x12\x19\n\x05label\x18\x02 \x01(\tH\x01R\x05label\x88\x01\x01\x12\x1d\n\x07message\x18\x03 \x01(\tH\x02R\x07message\x88\x01\x01\x12!\n\tsatisfied\x18\x04 \x01(\x08H\x03R\tsatisfied\x88\x01\x01\x12\x17\n\x04time\x18\x05 \x01(\x01H\x04R\x04time\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xde\n\n\x0bPbTaskProxy\x12\x19\n\x05stamp\x18\x01 \x01(\tH\x00R\x05stamp\x88\x01\x01\x12\x13\n\x02id\x18\x02 \x01(\tH\x01R\x02id\x88\x01\x01\x12\x17\n\x04task\x18\x03 \x01(\tH\x02R\x04task\x88\x01\x01\x12\x19\n\x05state\x18\x04 \x01(\tH\x03R\x05state\x88\x01\x01\x12$\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04R\ncyclePoint\x88\x01\x01\x12\x19\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05R\x05\x64\x65pth\x88\x01\x01\x12$\n\x0bjob_submits\x18\x07 \x01(\x05H\x06R\njobSubmits\x88\x01\x01\x12;\n\x07outputs\x18\t \x03(\x0b\x32!.cylc.v5.PbTaskProxy.OutputsEntryR\x07outputs\x12\x1c\n\tnamespace\x18\x0b \x03(\tR\tnamespace\x12=\n\rprerequisites\x18\x0c \x03(\x0b\x32\x17.cylc.v5.PbPrerequisiteR\rprerequisites\x12\x12\n\x04jobs\x18\r \x03(\tR\x04jobs\x12&\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07R\x0b\x66irstParent\x88\x01\x01\x12\x17\n\x04name\x18\x10 \x01(\tH\x08R\x04name\x88\x01\x01\x12\x1c\n\x07is_held\x18\x11 \x01(\x08H\tR\x06isHeld\x88\x01\x01\x12\x14\n\x05\x65\x64ges\x18\x12 \x03(\tR\x05\x65\x64ges\x12\x1c\n\tancestors\x18\x13 \x03(\tR\tancestors\x12 \n\tflow_nums\x18\x14 \x01(\tH\nR\x08\x66lowNums\x88\x01\x01\x12W\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32*.cylc.v5.PbTaskProxy.ExternalTriggersEntryR\x10\x65xternalTriggers\x12\x41\n\txtriggers\x18\x18 \x03(\x0b\x32#.cylc.v5.PbTaskProxy.XtriggersEntryR\txtriggers\x12 \n\tis_queued\x18\x19 \x01(\x08H\x0bR\x08isQueued\x88\x01\x01\x12$\n\x0bis_runahead\x18\x1a \x01(\x08H\x0cR\nisRunahead\x88\x01\x01\x12 \n\tflow_wait\x18\x1b \x01(\x08H\rR\x08\x66lowWait\x88\x01\x01\x12\x31\n\x07runtime\x18\x1c \x01(\x0b\x32\x12.cylc.v5.PbRuntimeH\x0eR\x07runtime\x88\x01\x01\x12$\n\x0bgraph_depth\x18\x1d \x01(\x05H\x0fR\ngraphDepth\x88\x01\x01\x1aM\n\x0cOutputsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x11.cylc.v5.PbOutputR\x05value:\x02\x38\x01\x1aW\n\x15\x45xternalTriggersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x12.cylc.v5.PbTriggerR\x05value:\x02\x38\x01\x1aP\n\x0eXtriggersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x12.cylc.v5.PbTriggerR\x05value:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xb9\x03\n\x08PbFamily\x12\x19\n\x05stamp\x18\x01 \x01(\tH\x00R\x05stamp\x88\x01\x01\x12\x13\n\x02id\x18\x02 \x01(\tH\x01R\x02id\x88\x01\x01\x12\x17\n\x04name\x18\x03 \x01(\tH\x02R\x04name\x88\x01\x01\x12(\n\x04meta\x18\x04 \x01(\x0b\x32\x0f.cylc.v5.PbMetaH\x03R\x04meta\x88\x01\x01\x12\x19\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04R\x05\x64\x65pth\x88\x01\x01\x12\x18\n\x07proxies\x18\x06 \x03(\tR\x07proxies\x12\x18\n\x07parents\x18\x07 \x03(\tR\x07parents\x12\x1f\n\x0b\x63hild_tasks\x18\x08 \x03(\tR\nchildTasks\x12%\n\x0e\x63hild_families\x18\t \x03(\tR\rchildFamilies\x12&\n\x0c\x66irst_parent\x18\n \x01(\tH\x05R\x0b\x66irstParent\x88\x01\x01\x12\x31\n\x07runtime\x18\x0b \x01(\x0b\x32\x12.cylc.v5.PbRuntimeH\x06R\x07runtime\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xa5\x08\n\rPbFamilyProxy\x12\x19\n\x05stamp\x18\x01 \x01(\tH\x00R\x05stamp\x88\x01\x01\x12\x13\n\x02id\x18\x02 \x01(\tH\x01R\x02id\x88\x01\x01\x12$\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02R\ncyclePoint\x88\x01\x01\x12\x17\n\x04name\x18\x04 \x01(\tH\x03R\x04name\x88\x01\x01\x12\x1b\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04R\x06\x66\x61mily\x88\x01\x01\x12\x19\n\x05state\x18\x06 \x01(\tH\x05R\x05state\x88\x01\x01\x12\x19\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06R\x05\x64\x65pth\x88\x01\x01\x12&\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07R\x0b\x66irstParent\x88\x01\x01\x12\x1f\n\x0b\x63hild_tasks\x18\n \x03(\tR\nchildTasks\x12%\n\x0e\x63hild_families\x18\x0b \x03(\tR\rchildFamilies\x12\x1c\n\x07is_held\x18\x0c \x01(\x08H\x08R\x06isHeld\x88\x01\x01\x12\x1c\n\tancestors\x18\r \x03(\tR\tancestors\x12\x16\n\x06states\x18\x0e \x03(\tR\x06states\x12J\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\'.cylc.v5.PbFamilyProxy.StateTotalsEntryR\x0bstateTotals\x12\'\n\ris_held_total\x18\x10 \x01(\x05H\tR\x0bisHeldTotal\x88\x01\x01\x12 \n\tis_queued\x18\x11 \x01(\x08H\nR\x08isQueued\x88\x01\x01\x12+\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0bR\risQueuedTotal\x88\x01\x01\x12$\n\x0bis_runahead\x18\x13 \x01(\x08H\x0cR\nisRunahead\x88\x01\x01\x12/\n\x11is_runahead_total\x18\x14 \x01(\x05H\rR\x0fisRunaheadTotal\x88\x01\x01\x12\x31\n\x07runtime\x18\x15 \x01(\x0b\x32\x12.cylc.v5.PbRuntimeH\x0eR\x07runtime\x88\x01\x01\x12$\n\x0bgraph_depth\x18\x16 \x01(\x05H\x0fR\ngraphDepth\x88\x01\x01\x1a>\n\x10StateTotalsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\x05R\x05value:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xe6\x01\n\x06PbEdge\x12\x19\n\x05stamp\x18\x01 \x01(\tH\x00R\x05stamp\x88\x01\x01\x12\x13\n\x02id\x18\x02 \x01(\tH\x01R\x02id\x88\x01\x01\x12\x1b\n\x06source\x18\x03 \x01(\tH\x02R\x06source\x88\x01\x01\x12\x1b\n\x06target\x18\x04 \x01(\tH\x03R\x06target\x88\x01\x01\x12\x1d\n\x07suicide\x18\x05 \x01(\x08H\x04R\x07suicide\x88\x01\x01\x12\x17\n\x04\x63ond\x18\x06 \x01(\x08H\x05R\x04\x63ond\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"\xb2\x01\n\x07PbEdges\x12\x13\n\x02id\x18\x01 \x01(\tH\x00R\x02id\x88\x01\x01\x12\x14\n\x05\x65\x64ges\x18\x02 \x03(\tR\x05\x65\x64ges\x12I\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x13.cylc.v5.PbPollTaskR\x14workflowPollingTasks\x12\x16\n\x06leaves\x18\x04 \x03(\tR\x06leaves\x12\x12\n\x04\x66\x65\x65t\x18\x05 \x03(\tR\x04\x66\x65\x65tB\x05\n\x03_id\"\xee\x02\n\x10PbEntireWorkflow\x12\x34\n\x08workflow\x18\x01 \x01(\x0b\x32\x13.cylc.v5.PbWorkflowH\x00R\x08workflow\x88\x01\x01\x12%\n\x05tasks\x18\x02 \x03(\x0b\x32\x0f.cylc.v5.PbTaskR\x05tasks\x12\x37\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x14.cylc.v5.PbTaskProxyR\x0btaskProxies\x12\"\n\x04jobs\x18\x04 \x03(\x0b\x32\x0e.cylc.v5.PbJobR\x04jobs\x12-\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\x11.cylc.v5.PbFamilyR\x08\x66\x61milies\x12=\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x16.cylc.v5.PbFamilyProxyR\rfamilyProxies\x12%\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x0f.cylc.v5.PbEdgeR\x05\x65\x64gesB\x0b\n\t_workflow\"\xf1\x01\n\x07\x45\x44\x65ltas\x12\x17\n\x04time\x18\x01 \x01(\x01H\x00R\x04time\x88\x01\x01\x12\x1f\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01R\x08\x63hecksum\x88\x01\x01\x12%\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0f.cylc.v5.PbEdgeR\x05\x61\x64\x64\x65\x64\x12)\n\x07updated\x18\x04 \x03(\x0b\x32\x0f.cylc.v5.PbEdgeR\x07updated\x12\x16\n\x06pruned\x18\x05 \x03(\tR\x06pruned\x12\x1f\n\x08reloaded\x18\x06 \x01(\x08H\x02R\x08reloaded\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xf5\x01\n\x07\x46\x44\x65ltas\x12\x17\n\x04time\x18\x01 \x01(\x01H\x00R\x04time\x88\x01\x01\x12\x1f\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01R\x08\x63hecksum\x88\x01\x01\x12\'\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x11.cylc.v5.PbFamilyR\x05\x61\x64\x64\x65\x64\x12+\n\x07updated\x18\x04 \x03(\x0b\x32\x11.cylc.v5.PbFamilyR\x07updated\x12\x16\n\x06pruned\x18\x05 \x03(\tR\x06pruned\x12\x1f\n\x08reloaded\x18\x06 \x01(\x08H\x02R\x08reloaded\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\x80\x02\n\x08\x46PDeltas\x12\x17\n\x04time\x18\x01 \x01(\x01H\x00R\x04time\x88\x01\x01\x12\x1f\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01R\x08\x63hecksum\x88\x01\x01\x12,\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x16.cylc.v5.PbFamilyProxyR\x05\x61\x64\x64\x65\x64\x12\x30\n\x07updated\x18\x04 \x03(\x0b\x32\x16.cylc.v5.PbFamilyProxyR\x07updated\x12\x16\n\x06pruned\x18\x05 \x03(\tR\x06pruned\x12\x1f\n\x08reloaded\x18\x06 \x01(\x08H\x02R\x08reloaded\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xef\x01\n\x07JDeltas\x12\x17\n\x04time\x18\x01 \x01(\x01H\x00R\x04time\x88\x01\x01\x12\x1f\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01R\x08\x63hecksum\x88\x01\x01\x12$\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.cylc.v5.PbJobR\x05\x61\x64\x64\x65\x64\x12(\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.cylc.v5.PbJobR\x07updated\x12\x16\n\x06pruned\x18\x05 \x03(\tR\x06pruned\x12\x1f\n\x08reloaded\x18\x06 \x01(\x08H\x02R\x08reloaded\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xf1\x01\n\x07TDeltas\x12\x17\n\x04time\x18\x01 \x01(\x01H\x00R\x04time\x88\x01\x01\x12\x1f\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01R\x08\x63hecksum\x88\x01\x01\x12%\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0f.cylc.v5.PbTaskR\x05\x61\x64\x64\x65\x64\x12)\n\x07updated\x18\x04 \x03(\x0b\x32\x0f.cylc.v5.PbTaskR\x07updated\x12\x16\n\x06pruned\x18\x05 \x03(\tR\x06pruned\x12\x1f\n\x08reloaded\x18\x06 \x01(\x08H\x02R\x08reloaded\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xfc\x01\n\x08TPDeltas\x12\x17\n\x04time\x18\x01 \x01(\x01H\x00R\x04time\x88\x01\x01\x12\x1f\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01R\x08\x63hecksum\x88\x01\x01\x12*\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x14.cylc.v5.PbTaskProxyR\x05\x61\x64\x64\x65\x64\x12.\n\x07updated\x18\x04 \x03(\x0b\x32\x14.cylc.v5.PbTaskProxyR\x07updated\x12\x16\n\x06pruned\x18\x05 \x03(\tR\x06pruned\x12\x1f\n\x08reloaded\x18\x06 \x01(\x08H\x02R\x08reloaded\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xfb\x01\n\x07WDeltas\x12\x17\n\x04time\x18\x01 \x01(\x01H\x00R\x04time\x88\x01\x01\x12.\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x13.cylc.v5.PbWorkflowH\x01R\x05\x61\x64\x64\x65\x64\x88\x01\x01\x12\x32\n\x07updated\x18\x03 \x01(\x0b\x32\x13.cylc.v5.PbWorkflowH\x02R\x07updated\x88\x01\x01\x12\x1f\n\x08reloaded\x18\x04 \x01(\x08H\x03R\x08reloaded\x88\x01\x01\x12\x1b\n\x06pruned\x18\x05 \x01(\tH\x04R\x06pruned\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xcd\x02\n\tAllDeltas\x12,\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x10.cylc.v5.FDeltasR\x08\x66\x61milies\x12\x38\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\x11.cylc.v5.FPDeltasR\rfamilyProxies\x12$\n\x04jobs\x18\x03 \x01(\x0b\x32\x10.cylc.v5.JDeltasR\x04jobs\x12&\n\x05tasks\x18\x04 \x01(\x0b\x32\x10.cylc.v5.TDeltasR\x05tasks\x12\x34\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\x11.cylc.v5.TPDeltasR\x0btaskProxies\x12&\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x10.cylc.v5.EDeltasR\x05\x65\x64ges\x12,\n\x08workflow\x18\x07 \x01(\x0b\x32\x10.cylc.v5.WDeltasR\x08workflowb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'cylc.v5.schema_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_PBWORKFLOW_STATETOTALSENTRY']._options = None + _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_options = b'8\001' + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._options = None + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_options = b'8\001' + _globals['_PBTASKPROXY_OUTPUTSENTRY']._options = None + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_options = b'8\001' + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._options = None + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_options = b'8\001' + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._options = None + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_options = b'8\001' + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._options = None + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_options = b'8\001' + _globals['_PBMETA']._serialized_start=34 + _globals['_PBMETA']._serialized_end=222 + _globals['_PBTIMEZONE']._serialized_start=225 + _globals['_PBTIMEZONE']._serialized_end=440 + _globals['_PBTASKPROXYREFS']._serialized_start=442 + _globals['_PBTASKPROXYREFS']._serialized_end=494 + _globals['_PBWORKFLOW']._serialized_start=497 + _globals['_PBWORKFLOW']._serialized_end=2635 + _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_start=2033 + _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_end=2095 + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_start=2097 + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_end=2190 + _globals['_PBRUNTIME']._serialized_start=2638 + _globals['_PBRUNTIME']._serialized_end=3768 + _globals['_PBJOB']._serialized_start=3771 + _globals['_PBJOB']._serialized_end=4630 + _globals['_PBTASK']._serialized_start=4633 + _globals['_PBTASK']._serialized_end=5101 + _globals['_PBPOLLTASK']._serialized_start=5104 + _globals['_PBPOLLTASK']._serialized_end=5378 + _globals['_PBCONDITION']._serialized_start=5381 + _globals['_PBCONDITION']._serialized_end=5636 + _globals['_PBPREREQUISITE']._serialized_start=5639 + _globals['_PBPREREQUISITE']._serialized_end=5845 + _globals['_PBOUTPUT']._serialized_start=5848 + _globals['_PBOUTPUT']._serialized_end=6021 + _globals['_PBTRIGGER']._serialized_start=6024 + _globals['_PBTRIGGER']._serialized_end=6226 + _globals['_PBTASKPROXY']._serialized_start=6229 + _globals['_PBTASKPROXY']._serialized_end=7603 + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=7153 + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=7230 + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=7232 + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=7319 + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=7321 + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=7401 + _globals['_PBFAMILY']._serialized_start=7606 + _globals['_PBFAMILY']._serialized_end=8047 + _globals['_PBFAMILYPROXY']._serialized_start=8050 + _globals['_PBFAMILYPROXY']._serialized_end=9111 + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_start=2033 + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_end=2095 + _globals['_PBEDGE']._serialized_start=9114 + _globals['_PBEDGE']._serialized_end=9344 + _globals['_PBEDGES']._serialized_start=9347 + _globals['_PBEDGES']._serialized_end=9525 + _globals['_PBENTIREWORKFLOW']._serialized_start=9528 + _globals['_PBENTIREWORKFLOW']._serialized_end=9894 + _globals['_EDELTAS']._serialized_start=9897 + _globals['_EDELTAS']._serialized_end=10138 + _globals['_FDELTAS']._serialized_start=10141 + _globals['_FDELTAS']._serialized_end=10386 + _globals['_FPDELTAS']._serialized_start=10389 + _globals['_FPDELTAS']._serialized_end=10645 + _globals['_JDELTAS']._serialized_start=10648 + _globals['_JDELTAS']._serialized_end=10887 + _globals['_TDELTAS']._serialized_start=10890 + _globals['_TDELTAS']._serialized_end=11131 + _globals['_TPDELTAS']._serialized_start=11134 + _globals['_TPDELTAS']._serialized_end=11386 + _globals['_WDELTAS']._serialized_start=11389 + _globals['_WDELTAS']._serialized_end=11640 + _globals['_ALLDELTAS']._serialized_start=11643 + _globals['_ALLDELTAS']._serialized_end=11976 +# @@protoc_insertion_point(module_scope) diff --git a/cylc/flow/data_messages_pb2.pyi b/cylc/flow/network/protobuf/cylc/v5/schema_pb2.pyi similarity index 100% rename from cylc/flow/data_messages_pb2.pyi rename to cylc/flow/network/protobuf/cylc/v5/schema_pb2.pyi diff --git a/cylc/flow/network/publisher.py b/cylc/flow/network/publisher.py index 70d40d3cdb9..78574f9e8c5 100644 --- a/cylc/flow/network/publisher.py +++ b/cylc/flow/network/publisher.py @@ -21,7 +21,7 @@ import zmq from cylc.flow import LOG -from cylc.flow.network import ZMQSocketBase +from cylc.flow.network.base import ZMQSocketBase def serialize_data( diff --git a/cylc/flow/network/replier.py b/cylc/flow/network/replier.py index 09bfb55f662..a40756c05b4 100644 --- a/cylc/flow/network/replier.py +++ b/cylc/flow/network/replier.py @@ -21,7 +21,8 @@ import zmq from cylc.flow import LOG -from cylc.flow.network import encode_, decode_, ZMQSocketBase +from cylc.flow.network.base import ZMQSocketBase +from cylc.flow.network.util import encode_, decode_ if TYPE_CHECKING: from cylc.flow.network.server import WorkflowRuntimeServer diff --git a/cylc/flow/network/server.py b/cylc/flow/network/server.py index 2c170e61198..c7e62dbe8b2 100644 --- a/cylc/flow/network/server.py +++ b/cylc/flow/network/server.py @@ -36,7 +36,7 @@ from cylc.flow.network.resolvers import Resolvers from cylc.flow.network.schema import schema from cylc.flow.data_store_mgr import DELTAS_MAP -from cylc.flow.data_messages_pb2 import PbEntireWorkflow +from cylc.flow.network.protobuf.cylc.v5.schema_pb2 import PbEntireWorkflow if TYPE_CHECKING: from cylc.flow.scheduler import Scheduler diff --git a/cylc/flow/network/subscriber.py b/cylc/flow/network/subscriber.py index 66bd16f81f8..28d5b5d1bb2 100644 --- a/cylc/flow/network/subscriber.py +++ b/cylc/flow/network/subscriber.py @@ -22,8 +22,9 @@ import zmq -from cylc.flow.network import ZMQSocketBase, get_location from cylc.flow.data_store_mgr import DELTAS_MAP +from cylc.flow.network.base import ZMQSocketBase +from cylc.flow.network.util import get_location if TYPE_CHECKING: import zmq.asyncio diff --git a/cylc/flow/network/util.py b/cylc/flow/network/util.py new file mode 100644 index 00000000000..6d1a006060d --- /dev/null +++ b/cylc/flow/network/util.py @@ -0,0 +1,77 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Common networking utilities.""" + +import getpass +import json +from typing import Tuple + +from cylc.flow.exceptions import ( + CylcVersionError, + ServiceFileError, + WorkflowStopped +) +from cylc.flow.hostuserutil import get_fqdn_by_host +from cylc.flow.workflow_files import ( + ContactFileFields, + load_contact_file, +) + + +def encode_(message): + """Convert the structure holding a message field from JSON to a string.""" + try: + return json.dumps(message) + except TypeError as exc: + return json.dumps({'errors': [{'message': str(exc)}]}) + + +def decode_(message): + """Convert an encoded message string to JSON with an added 'user' field.""" + msg = json.loads(message) + msg['user'] = getpass.getuser() # assume this is the user + return msg + + +def get_location(workflow: str) -> Tuple[str, int, int]: + """Extract host and port from a workflow's contact file. + + NB: if it fails to load the workflow contact file, it will exit. + + Args: + workflow: workflow ID + Returns: + Tuple (host name, port number, publish port number) + Raises: + WorkflowStopped: if the workflow is not running. + CylcVersionError: if target is a Cylc 7 (or earlier) workflow. + """ + try: + contact = load_contact_file(workflow) + except (IOError, ValueError, ServiceFileError): + # Contact file does not exist or corrupted, workflow should be dead + raise WorkflowStopped(workflow) + + host = contact[ContactFileFields.HOST] + host = get_fqdn_by_host(host) + port = int(contact[ContactFileFields.PORT]) + if ContactFileFields.PUBLISH_PORT in contact: + pub_port = int(contact[ContactFileFields.PUBLISH_PORT]) + else: + version = contact.get('CYLC_VERSION', None) + raise CylcVersionError(version=version) + return host, port, pub_port diff --git a/cylc/flow/prerequisite.py b/cylc/flow/prerequisite.py index b388934e2de..3220385adc3 100644 --- a/cylc/flow/prerequisite.py +++ b/cylc/flow/prerequisite.py @@ -22,7 +22,7 @@ from cylc.flow.cycling.loader import get_point from cylc.flow.exceptions import TriggerExpressionError -from cylc.flow.data_messages_pb2 import ( +from cylc.flow.network.protobuf.cylc.v5.schema_pb2 import ( PbPrerequisite, PbCondition, ) diff --git a/cylc/flow/scripts/subscribe.py b/cylc/flow/scripts/subscribe.py index 5d174718c23..bd42dd6c90d 100755 --- a/cylc/flow/scripts/subscribe.py +++ b/cylc/flow/scripts/subscribe.py @@ -34,7 +34,7 @@ WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, ) -from cylc.flow.network import get_location +from cylc.flow.network.util import get_location from cylc.flow.network.subscriber import WorkflowSubscriber, process_delta_msg from cylc.flow.terminal import cli_function from cylc.flow.data_store_mgr import DELTAS_MAP diff --git a/pytest.ini b/pytest.ini index 04d12d224b4..4abf7d7f9ee 100644 --- a/pytest.ini +++ b/pytest.ini @@ -22,7 +22,7 @@ addopts = --verbose # group tests by module or class --dist=loadscope # ignore files which cause issues with test collection - --ignore=cylc/flow/data_messages_pb2.py + --ignore=cylc/flow/v5/* --ignore=cylc/flow/parsec/empysupport.py --ignore=cylc/flow/parsec/example # disable pytest-tornasync because it conflicts with pytest-asyncio's auto mode diff --git a/tests/integration/test_replier.py b/tests/integration/test_replier.py index ce0b53fdaa8..7e219e8dd44 100644 --- a/tests/integration/test_replier.py +++ b/tests/integration/test_replier.py @@ -14,11 +14,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from async_timeout import timeout -from cylc.flow.network import decode_ -from cylc.flow.network.client import WorkflowRuntimeClient import asyncio +from cylc.flow.network.client import WorkflowRuntimeClient +from cylc.flow.network.util import decode_ + +from async_timeout import timeout import pytest diff --git a/tests/integration/test_zmq.py b/tests/integration/test_zmq.py index 24c8db6d9b0..41ef6be2767 100644 --- a/tests/integration/test_zmq.py +++ b/tests/integration/test_zmq.py @@ -18,7 +18,7 @@ import zmq from cylc.flow.exceptions import CylcError -from cylc.flow.network import ZMQSocketBase +from cylc.flow.network.base import ZMQSocketBase from .key_setup import setup_keys diff --git a/tests/unit/network/test_graphql.py b/tests/unit/network/test_graphql.py index e5be079e289..e4b741617c1 100644 --- a/tests/unit/network/test_graphql.py +++ b/tests/unit/network/test_graphql.py @@ -20,7 +20,7 @@ from pytest import param from graphql import parse -from cylc.flow.data_messages_pb2 import PbTaskProxy, PbPrerequisite +from cylc.flow.network.protobuf.cylc.v5.schema_pb2 import PbTaskProxy, PbPrerequisite from cylc.flow.network.graphql import ( AstDocArguments, null_setter, NULL_VALUE, grow_tree ) diff --git a/tests/unit/network/test__init__.py b/tests/unit/network/test_network_util.py similarity index 80% rename from tests/unit/network/test__init__.py rename to tests/unit/network/test_network_util.py index 71c32cf9bd1..1d40ca3b91e 100644 --- a/tests/unit/network/test__init__.py +++ b/tests/unit/network/test_network_util.py @@ -17,10 +17,10 @@ import pytest -import cylc +import cylc.flow.network.util from cylc.flow.exceptions import CylcVersionError -from cylc.flow.network import get_location -from cylc.flow.workflow_files import load_contact_file, ContactFileFields +from cylc.flow.network.util import get_location +from cylc.flow.workflow_files import ContactFileFields BASE_CONTACT_DATA = { @@ -33,7 +33,7 @@ def mpatch_get_fqdn_by_host(monkeypatch): """Monkeypatch function used the same by all tests.""" monkeypatch.setattr( - cylc.flow.network, 'get_fqdn_by_host', lambda _ : 'myhost.x.y.z' + cylc.flow.network.util, 'get_fqdn_by_host', lambda _: 'myhost.x.y.z' ) @@ -42,7 +42,7 @@ def test_get_location_ok(monkeypatch, mpatch_get_fqdn_by_host): contact_data = BASE_CONTACT_DATA.copy() contact_data[ContactFileFields.PUBLISH_PORT] = '8042' monkeypatch.setattr( - cylc.flow.network, 'load_contact_file', lambda _ : contact_data + cylc.flow.network.util, 'load_contact_file', lambda _: contact_data ) assert get_location('_') == ( 'myhost.x.y.z', 42, 8042 @@ -55,7 +55,7 @@ def test_get_location_old_contact_file(monkeypatch, mpatch_get_fqdn_by_host): contact_data['CYLC_SUITE_PUBLISH_PORT'] = '8042' contact_data['CYLC_VERSION'] = '5.1.2' monkeypatch.setattr( - cylc.flow.network, 'load_contact_file', lambda _ : contact_data + cylc.flow.network.util, 'load_contact_file', lambda _: contact_data ) - with pytest.raises(CylcVersionError, match=r'.*5.1.2.*') as exc: + with pytest.raises(CylcVersionError, match=r'.*5.1.2.*'): get_location('_') diff --git a/tox.ini b/tox.ini index d9954dbb7e8..52e1183cb2b 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ exclude= .git, __pycache__, .tox, - **data_messages_pb2.py + cylc/flow/network/protobuf/cylc/** paths = ./cylc/flow ./tests