feat(meta): add seq to ExpireValue (#11686)

* doc: Explain meta data crate and how to convert to protobuf

* doc(meta): update obsolete command line doc

* feat(meta): add `seq` to ExpireValue

`ExpireValue` is the value part of the expiration index of generic-kv in
meta-service. To support snapshot read(in next PR), it must store
a `seq` number to indicate the freshness of the index entry.

This is a **compatible** change to the meta data.

src/meta/README.md

@@ -14,3 +14,57 @@ Databend Meta is a transactional metadata service.
 - [`store`](./store/), impl with either a local embedded meta store, or a grpc-client of meta service.
 - [`types`](./types/): defines the rust types for metadata.
 - [`ee`](./ee/) contains enterprise functionalities.
+
+
+## How to add new meta data types to store in meta-service
+
+Databend meta-service stores raw bytes and does not understand what the bytes are.
+
+Databend-query use rust types in its runtime, these types such as `TableMeta`
+must be serialized to be stored in meta-service.
+
+The serialization is implemented with `protobuf` and a protobuf message provides
+the backward compatibility, i.e., a newer version(version-B) protobuf message can be deserialized
+from an older version(version-A) of serialized bytes, and version-B protobuf
+message can be converted to version-B rust types.
+
+- Rust types are defined in `src/meta/app/src/`,  such as `TableMeta` that is
+  defined in `src/meta/app/src/schema/table.rs`.
+
+- The corresponding protobuf message is defined in `src/meta/protos/proto/`,
+  such as `src/meta/protos/proto/table.proto`.
+
+- The conversion between protobuf message and rust type is defined in
+  `src/meta/proto-conv/`, such as
+  `src/meta/proto-conv/src/table_from_to_protobuf_impl.rs`,
+  by implementing a `FromToProto` trait.
+
+To add a new feature(add new type or update an type), the developer should do:
+
+- Add the rust types, in one mod in the `src/meta/app/src/`;
+
+- Add a new version in `src/meta/proto-conv/src/util.rs`. The versions track
+    change history and will be checked when converting protobuf message to rust
+    types:
+
+    ```rust
+    const META_CHANGE_LOG: &[(u64, &str)] = &[
+        //
+        ( 1, "----------: Initial", ),
+        ( 2, "2022-07-13: Add: share.proto", ),
+        ( 3, "2022-07-29: Add: user.proto/UserOption::default_role", ),
+        ...
+        (37, "2023-05-05: Add: index.proto", ),
+        (38, "2023-05-19: Rename: table.proto/TableCopiedFileLock to EmptyProto", ),
+        (39, "2023-05-22: Add: data_mask.proto", ),
+    ];
+    ```
+
+    Note that only add new version to the bottom and remove old version from the
+    top.
+
+- Add the conversion implementation to `src/meta/proto-conv/src/`, refer to
+    other files in this crate.
+
+- Add a compatibility test to ensure that compatibility will always be kept in
+    future, a good example is: `src/meta/proto-conv/tests/it/v039_data_mask.rs`

src/meta/raft-store/src/state_machine/expire.rs

@@ -45,11 +45,18 @@ pub struct ExpireKey {
 }
 
 /// The value of an expiration index is the record key.
-#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Default, Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub struct ExpireValue {
+    #[serde(skip_serializing_if = "is_zero")]
+    #[serde(default)]
+    pub seq: u64,
     pub key: String,
 }
 
+fn is_zero(v: &u64) -> bool {
+    *v == 0
+}
+
 impl SledSerde for ExpireValue {
     fn de<T: AsRef<[u8]>>(v: T) -> Result<Self, SledBytesError>
     where Self: Sized {
@@ -58,6 +65,15 @@ impl SledSerde for ExpireValue {
     }
 }
 
+impl ExpireValue {
+    pub fn new(key: impl ToString, seq: u64) -> Self {
+        Self {
+            key: key.to_string(),
+            seq,
+        }
+    }
+}
+
 impl SledOrderedSerde for ExpireKey {
     fn ser(&self) -> Result<IVec, SledBytesError> {
         let size = size_of_val(self);
@@ -98,6 +114,7 @@ mod tests {
     use common_meta_sled_store::SledOrderedSerde;
 
     use crate::state_machine::ExpireKey;
+    use crate::state_machine::ExpireValue;
 
     #[test]
     fn test_expire_key_serde() -> anyhow::Result<()> {
@@ -125,4 +142,35 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn test_expire_value_serde() -> anyhow::Result<()> {
+        {
+            let v = ExpireValue {
+                seq: 0,
+                key: "a".to_string(),
+            };
+            let s = serde_json::to_string(&v)?;
+            let want = r#"{"key":"a"}"#;
+            assert_eq!(want, s);
+
+            let got = serde_json::from_str::<ExpireValue>(want)?;
+            assert_eq!(v, got);
+        }
+
+        {
+            let v = ExpireValue {
+                seq: 5,
+                key: "a".to_string(),
+            };
+            let s = serde_json::to_string(&v)?;
+            let want = r#"{"seq":5,"key":"a"}"#;
+            assert_eq!(want, s);
+
+            let got = serde_json::from_str::<ExpireValue>(want)?;
+            assert_eq!(v, got);
+        }
+
+        Ok(())
+    }
 }

src/meta/raft-store/src/state_machine/sm.rs

@@ -948,9 +948,7 @@ impl StateMachine {
             if let Some(m) = &sv.meta {
                 if let Some(exp) = m.expire_at {
                     let k = ExpireKey::new(exp * 1000, sv.seq);
-                    let v = ExpireValue {
-                        key: upsert_kv.key.clone(),
-                    };
+                    let v = ExpireValue::new(&upsert_kv.key, 0);
                     expires.insert(&k, &v)?;
                 }
             }

src/meta/service/src/configs/outer_v0.rs

@@ -450,13 +450,16 @@ pub struct RaftConfig {
     #[clap(long, default_value = "1000")]
     pub max_applied_log_to_keep: u64,
 
-    /// Single node metasrv. It creates a single node cluster if meta data is not initialized.
-    /// Otherwise it opens the previous one.
-    /// This is mainly for testing purpose.
+    /// Start databend-meta in single node mode.
+    /// It initialize a single node cluster, if meta data is not initialized.
+    /// If on-disk data is already initialized, this argument has no effect.
     #[clap(long)]
     pub single: bool,
 
-    /// Bring up a metasrv node and join a cluster.
+    /// Bring up a databend-meta node and join a cluster.
+    ///
+    /// It will take effect only when the meta data is not initialized.
+    /// If on-disk data is already initialized, this argument has no effect.
     ///
     /// The value is one or more addresses of a node in the cluster, to which this node sends a `join` request.
     #[clap(long, multiple_occurrences = true, multiple_values = true)]
@@ -473,7 +476,7 @@ pub struct RaftConfig {
     pub leave_id: Option<u64>,
 
     /// The node id. Only used when this server is not initialized,
-    ///  e.g. --boot or --single for the first time.
+    ///  e.g. --single for the first time.
     ///  Otherwise this argument is ignored.
     #[clap(long, default_value = "0")]
     pub id: u64,