protobuf: automate derived file generation

* Use the bufbuild/buf tool to:
  * Lint the protobuf schema.
  * Check for breaking changes.
  * Configure the generation of derived Python files.
* Adds a GitHub action which automates the checking and generation of
  derived files and commits the results back on pull request branches.

.codacy.yml

@@ -1,4 +1,4 @@
 exclude_paths:
   - 'etc/**'
   - 'tests/**'
-  - 'cylc/flow/**_pb2.py'
+  - 'cylc/flow/network/protobuf/**_pb2.py'

.codecov.yml

@@ -28,7 +28,7 @@ coverage:
 # files to ignore
 ignore:
   - "tests/**"
-  - "ws_messages_pb2.py"
+  - "cylc/flow/network/protobuf/cylc/v5/schema_pb2.py"
   - "cylc/flow/scripts/report_timings.py"
 
 flag_management:

.coveragerc

@@ -14,7 +14,7 @@ disable_warnings =
     module-not-measured
 omit =
     tests/*
-    */cylc/flow/*_pb2.py
+    */cylc/flow/network/protobuf/cylc/v5/*_pb2.py
     cylc/flow/etc/*
     cylc/flow/scripts/report_timings.py
 parallel = True
@@ -43,7 +43,7 @@ fail_under=0
 ignore_errors = False
 omit =
     tests/*
-    */cylc/flow/*_pb2.py
+    */cylc/flow/network/protobuf/cylc/v5/*_pb2.py
     cylc/flow/etc/*
 precision = 2
 show_missing = False

.github/workflows/protobuf.yml

@@ -0,0 +1,82 @@
+name: protobuf
+
+on:
+  pull_request:
+  # NOTE: don't run this on "workflow_dispatch" or "schedule" as it will commit
+  # directly to the branch
+
+jobs:
+  protobuf:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # we need all of the commits on the PR branch in order to be able to add new ones
+          fetch-depth: 100
+
+      - name: Configure git
+        uses: cylc/release-actions/configure-git@v1
+
+      - name: Install Protobuf
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-name: protobuf
+          create-args: protobuf
+          init-shell: bash
+
+      - name: Install bufbuild/buf
+        run: |
+          # activate homebrew (NOTE: must use "shell: bash -el {0}" to use this)
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"  # activate homebrew
+
+          # NOTE: bufbuild does exist on conda-forge but hasn't been updated for a while
+          brew install bufbuild/buf/buf
+
+      - name: Lint
+        run: |
+          # lint .proto files
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"  # activate homebrew
+          cd cylc/flow/network/protobuf
+          buf lint
+
+      - name: Compatibility
+        shell: bash -el {0}
+        run: |
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"  # activate homebrew
+          cd cylc/flow/network/protobuf
+          # NOTE: "buf breaking" exits "100" for a missing file (i.e. new version not present on old branch)
+            # --against 'https://github.com/cylc/cylc-flow.git#tag=${{ github.base_ref }},subdir=proto') \
+          buf breaking \
+            --against 'https://github.com/oliver-sanders/cylc-flow.git#tag=${{ github.base_ref }},subdir=cylc/flow/network/protobuf' \
+            || ( \
+              ret=$?; \
+              if [[ $ret == 100 ]]; then exit 0; \
+              elif [[ $ret != 0 ]]; then echo '::error:: Breaking changes detected, create a new version of the protobuf schema if this is desired and increment the Cylc API number to match'; \
+              fi \
+            )
+
+      - name: Build
+        shell: bash -el {0}
+        run: |
+          # generate .py and .pyi files from the .proto files
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"  # activate homebrew
+          micromamba activate protobuf
+          cd cylc/flow/network/protobuf
+          buf generate
+
+      - name: Commit & Push
+        run: |
+          if [[ -z $(git diff --stat) ]]; then
+            echo '::error:: No functional changes made to the protobuf schema'
+            exit 0
+          else
+            echo '::info:: pushing update commit'
+              git add -u
+              git commit -m 'protobuf: updating generated files'
+              git remote add pr https://github.com/${{ github.event.pull_request.head.repo.owner.login }}/cylc-flow
+              git push pr HEAD:${{ github.head_ref }}
+            exit 0
+          fi

cylc/flow/data_store_mgr.py

@@ -73,7 +73,7 @@
 
 from cylc.flow import __version__ as CYLC_VERSION, LOG
 from cylc.flow.cycling.loader import get_point
-from cylc.flow.data_messages_pb2 import (
+from cylc.flow.network.protobuf.cylc.v5.schema_pb2 import (
     PbEdge, PbEntireWorkflow, PbFamily, PbFamilyProxy, PbJob, PbTask,
     PbTaskProxy, PbWorkflow, PbRuntime, AllDeltas, EDeltas, FDeltas,
     FPDeltas, JDeltas, TDeltas, TPDeltas, WDeltas)
@@ -382,7 +382,7 @@ def create_delta_store(delta=None, workflow_id=None):
     """Create a mini data-store out of the all deltas message.
 
     Args:
-        delta (cylc.flow.data_messages_pb2.AllDeltas):
+        delta (cylc.flow.v5.schema_pb2.AllDeltas):
             The message of accumulated deltas for publish/push.
         workflow_id (str):
             The workflow ID.
@@ -430,18 +430,18 @@ class DataStoreMgr:
             Local store of config.get_first_parent_ancestors()
         .data (dict):
             .edges (dict):
-                cylc.flow.data_messages_pb2.PbEdge by internal ID.
+                cylc.flow.v5.schema_pb2.PbEdge by internal ID.
             .families (dict):
-                cylc.flow.data_messages_pb2.PbFamily by name (internal ID).
+                cylc.flow.v5.schema_pb2.PbFamily by name (internal ID).
             .family_proxies (dict):
-                cylc.flow.data_messages_pb2.PbFamilyProxy by internal ID.
+                cylc.flow.v5.schema_pb2.PbFamilyProxy by internal ID.
             .jobs (dict):
-                cylc.flow.data_messages_pb2.PbJob by internal ID.
+                cylc.flow.v5.schema_pb2.PbJob by internal ID.
             .tasks (dict):
-                cylc.flow.data_messages_pb2.PbTask by name (internal ID).
+                cylc.flow.v5.schema_pb2.PbTask by name (internal ID).
             .task_proxies (dict):
-                cylc.flow.data_messages_pb2.PbTaskProxy by internal ID.
-            .workflow (cylc.flow.data_messages_pb2.PbWorkflow)
+                cylc.flow.v5.schema_pb2.PbTaskProxy by internal ID.
+            .workflow (cylc.flow.v5.schema_pb2.PbWorkflow)
                 Message containing the global information of the workflow.
         .descendants (dict):
             Local store of config.get_first_parent_descendants()
@@ -2688,7 +2688,7 @@ def get_entire_workflow(self):
         """Gather data elements into single Protobuf message.
 
         Returns:
-            cylc.flow.data_messages_pb2.PbEntireWorkflow
+            cylc.flow.v5.schema_pb2.PbEntireWorkflow
 
         """
 

cylc/flow/network/protobuf/buf.gen.yaml

@@ -0,0 +1,10 @@
+# file containing protobuf configuration
+
+version: v2
+
+plugins:
+  - protoc_builtin: python
+    out: .
+
+  - protoc_builtin: pyi
+    out: .

cylc/flow/network/protobuf/buf.yaml

@@ -0,0 +1,11 @@
+# file containing protobuf configuration
+
+version: v2
+
+lint:
+  use:
+    - DEFAULT
+
+breaking:
+  use:
+    - FILE