feat: Serialization upgrade path (#1327)

This PR does three things:
- makes changes to serialization code necessary to allow
backwards-compatible upgrades;
- makes a small change to serialization (changing "op_name" to "name" in
`CustomOp`s), adding v2 of the serialization format;
- adds documentation in `DEVELOPMENT.md` on how to make upgrades in the
future.

DEVELOPMENT.md

@@ -110,6 +110,38 @@ just coverage
 and open it with your favourite coverage viewer. In VSCode, you can use
 [`coverage-gutters`](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters).
 
+## Serialization
+
+If you want to make a change that modifies the serialization schema, you must
+ensure backwards-compatibility by writing a method to convert from the existing
+format to the new one. We suggest the following process. (For concreteness we
+assume that you are upgrading from v5 to v6.)
+
+1.  Add a test case in `hugr-core/src/hugr/serialize/upgrade/test.rs` that
+    exercises the part of the schema that will change in v6.
+2.  Run the tests. This will create a new JSON file in the `testcases`
+    subdirectory. Commit this to the repo.
+3.  Implement the schema-breaking change. Expect the test you added in step 1
+    (and possibly others) to fail.
+4.  In `hugr/hugr-core/src/hugr/serialize.rs`:
+    - Add a new line `V6(SerHugr),` in `enum Versioned`, and change the previous
+      line to `V5(serde_json::Value),`.
+    - In `Versioned::upgrade()` insert the line
+      `Self::V5(json) => self = Self::V6(upgrade::v5_to_v6(json).and_then(go)?),`
+      and change `V5` to `V6` in the line
+      `Self::V5(ser_hugr) => return Ok(ser_hugr),`.
+    - Change `new_latest()` to return `Self::V6(t)`.
+5.  In `hugr-core/src/hugr/serialize/upgrade.rs` add a stub implementation of
+    `v5_to_v6()`.
+6.  In `hugr-py/src/hugr/__init__.py` update `get_serialisation_version()` to
+    return `"v6"`.
+7.  Run `just update-schema` to generate new v6 schema files. Commit these to
+    the repo.
+8.  In `hugr-core/src/hugr/serialize/test.rs`, in the `include_schema` macro
+    change `v5` to `v6`.
+9.  Implement `v5_to_v6()`.
+10. Ensure all tests are passing.
+
 ## 🌐 Contributing to HUGR
 
 We welcome contributions to HUGR! Please open [an issue](https://github.com/CQCL/hugr/issues/new) or [pull request](https://github.com/CQCL/hugr/compare) if you have any questions or suggestions.

hugr-core/src/extension.rs

@@ -301,8 +301,8 @@ impl Extension {
     }
 
     /// Allows read-only access to the operations in this Extension
-    pub fn get_op(&self, op_name: &OpNameRef) -> Option<&Arc<op_def::OpDef>> {
-        self.operations.get(op_name)
+    pub fn get_op(&self, name: &OpNameRef) -> Option<&Arc<op_def::OpDef>> {
+        self.operations.get(name)
     }
 
     /// Allows read-only access to the types in this Extension
@@ -352,11 +352,11 @@ impl Extension {
     /// Instantiate an [`ExtensionOp`] which references an [`OpDef`] in this extension.
     pub fn instantiate_extension_op(
         &self,
-        op_name: &OpNameRef,
+        name: &OpNameRef,
         args: impl Into<Vec<TypeArg>>,
         ext_reg: &ExtensionRegistry,
     ) -> Result<ExtensionOp, SignatureError> {
-        let op_def = self.get_op(op_name).expect("Op not found.");
+        let op_def = self.get_op(name).expect("Op not found.");
         ExtensionOp::new(op_def.clone(), args, ext_reg)
     }
 

hugr-core/src/hugr/serialize.rs

@@ -1,6 +1,7 @@
 //! Serialization definition for [`Hugr`]
 //! [`Hugr`]: crate::hugr::Hugr
 
+use serde::de::DeserializeOwned;
 use std::collections::HashMap;
 use thiserror::Error;
 
@@ -13,8 +14,12 @@ use portgraph::{Direction, LinkError, PortView};
 
 use serde::{Deserialize, Deserializer, Serialize};
 
+use self::upgrade::UpgradeError;
+
 use super::{HugrMut, HugrView, NodeMetadataMap};
 
+mod upgrade;
+
 /// A wrapper over the available HUGR serialization formats.
 ///
 /// The implementation of `Serialize` for `Hugr` encodes the graph in the most
@@ -26,21 +31,44 @@ use super::{HugrMut, HugrView, NodeMetadataMap};
 ///
 /// Make sure to order the variants from newest to oldest, as the deserializer
 /// will try to deserialize them in order.
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(tag = "version", rename_all = "lowercase")]
-enum Versioned<SerHugr> {
+enum Versioned<SerHugr = SerHugrLatest> {
+    #[serde(skip_serializing)]
     /// Version 0 of the HUGR serialization format.
     V0,
-    /// Version 1 of the HUGR serialization format.
-    V1(SerHugr),
 
+    V1(serde_json::Value),
+    V2(SerHugr),
+
+    #[serde(skip_serializing)]
     #[serde(other)]
     Unsupported,
 }
 
 impl<T> Versioned<T> {
-    pub fn new(t: T) -> Self {
-        Self::V1(t)
+    pub fn new_latest(t: T) -> Self {
+        Self::V2(t)
+    }
+}
+
+impl<T: DeserializeOwned> Versioned<T> {
+    fn upgrade(mut self) -> Result<T, UpgradeError> {
+        // go is polymorphic in D. When we are upgrading to the latest version
+        // D is T. When we are upgrading to a version which is not the latest D
+        // is serde_json::Value.
+        fn go<D: serde::de::DeserializeOwned>(v: serde_json::Value) -> Result<D, UpgradeError> {
+            serde_json::from_value(v).map_err(Into::into)
+        }
+        loop {
+            match self {
+                Self::V0 => Err(UpgradeError::KnownVersionUnsupported("0".into()))?,
+                // the upgrade lines remain unchanged when adding a new constructor
+                Self::V1(json) => self = Self::V2(upgrade::v1_to_v2(json).and_then(go)?),
+                Self::V2(ser_hugr) => return Ok(ser_hugr),
+                Versioned::Unsupported => Err(UpgradeError::UnknownVersionUnsupported)?,
+            }
+        }
     }
 }
 
@@ -52,8 +80,8 @@ struct NodeSer {
 }
 
 /// Version 1 of the HUGR serialization format.
-#[derive(Serialize, Deserialize, PartialEq, Debug)]
-struct SerHugrV1 {
+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]
+struct SerHugrLatest {
     /// For each node: (parent, node_operation)
     nodes: Vec<NodeSer>,
     /// for each edge: (src, src_offset, tgt, tgt_offset)
@@ -102,8 +130,8 @@ impl Serialize for Hugr {
     where
         S: serde::Serializer,
     {
-        let shg: SerHugrV1 = self.try_into().map_err(serde::ser::Error::custom)?;
-        let versioned = Versioned::new(shg);
+        let shg: SerHugrLatest = self.try_into().map_err(serde::ser::Error::custom)?;
+        let versioned = Versioned::new_latest(shg);
         versioned.serialize(serializer)
     }
 }
@@ -113,20 +141,13 @@ impl<'de> Deserialize<'de> for Hugr {
     where
         D: Deserializer<'de>,
     {
-        let shg: Versioned<SerHugrV1> = Versioned::deserialize(deserializer)?;
-        match shg {
-            Versioned::V0 => Err(serde::de::Error::custom(
-                "Version 0 HUGR serialization format is not supported.",
-            )),
-            Versioned::V1(shg) => shg.try_into().map_err(serde::de::Error::custom),
-            Versioned::Unsupported => Err(serde::de::Error::custom(
-                "Unsupported HUGR serialization format.",
-            )),
-        }
+        let versioned = Versioned::deserialize(deserializer)?;
+        let shl: SerHugrLatest = versioned.upgrade().map_err(serde::de::Error::custom)?;
+        shl.try_into().map_err(serde::de::Error::custom)
     }
 }
 
-impl TryFrom<&Hugr> for SerHugrV1 {
+impl TryFrom<&Hugr> for SerHugrLatest {
     type Error = HUGRSerializationError;
 
     fn try_from(hugr: &Hugr) -> Result<Self, Self::Error> {
@@ -188,15 +209,15 @@ impl TryFrom<&Hugr> for SerHugrV1 {
     }
 }
 
-impl TryFrom<SerHugrV1> for Hugr {
+impl TryFrom<SerHugrLatest> for Hugr {
     type Error = HUGRSerializationError;
     fn try_from(
-        SerHugrV1 {
+        SerHugrLatest {
             nodes,
             edges,
             metadata,
-            ..
-        }: SerHugrV1,
+            encoder: _,
+        }: SerHugrLatest,
     ) -> Result<Self, Self::Error> {
         // Root must be first node
         let mut nodes = nodes.into_iter();