diff --git a/.github/actions/test_compat/action.yml b/.github/actions/test_compat/action.yml
index 2710d49cf5d33..b86fa7baed96e 100644
--- a/.github/actions/test_compat/action.yml
+++ b/.github/actions/test_compat/action.yml
@@ -26,7 +26,7 @@ runs:
     - uses: actions/download-artifact@v2
       with:
         name: ${{ inputs.profile }}-${{ github.sha }}-${{ inputs.target }}
-        path: ./current/
+        path: ./bins/current/
 
     - uses: dsaltares/fetch-gh-release-asset@master
       with:
diff --git a/Cargo.lock b/Cargo.lock
index 3c055e88d1873..be7d852077e91 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1141,6 +1141,7 @@ version = "0.1.0"
 dependencies = [
  "common-arrow",
  "common-base",
+ "common-building",
  "common-exception",
  "common-grpc",
  "common-meta-api",
@@ -1150,8 +1151,10 @@ dependencies = [
  "common-tracing",
  "derive_more",
  "futures",
+ "once_cell",
  "prost 0.9.0",
  "rand 0.8.5",
+ "semver",
  "serde",
  "serde_json",
  "thiserror",
@@ -1743,6 +1746,7 @@ dependencies = [
  "clap 3.1.8",
  "common-arrow",
  "common-base",
+ "common-building",
  "common-exception",
  "common-grpc",
  "common-macros",
@@ -1764,6 +1768,7 @@ dependencies = [
  "prost 0.9.0",
  "regex",
  "reqwest",
+ "semver",
  "serde",
  "serde-bridge",
  "serde_json",
@@ -1873,6 +1878,7 @@ dependencies = [
  "regex",
  "reqwest",
  "rsa",
+ "semver",
  "serde",
  "serde-bridge",
  "serde_json",
@@ -5624,9 +5630,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.4"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012"
+checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd"
 dependencies = [
  "serde",
 ]
diff --git a/common/exception/src/exception_code.rs b/common/exception/src/exception_code.rs
index f47b683972956..dafcf37034dc6 100644
--- a/common/exception/src/exception_code.rs
+++ b/common/exception/src/exception_code.rs
@@ -158,6 +158,7 @@ build_exceptions! {
     MetaServiceError(2001),
     InvalidConfig(2002),
     MetaStorageError(2003),
+    InvalidArgument(2004),
 
     TableVersionMismatched(2009),
     OCCRetryFailure(2011),
diff --git a/common/meta/grpc/Cargo.toml b/common/meta/grpc/Cargo.toml
index 9c8dafb7dca3a..2c1d60322e844 100644
--- a/common/meta/grpc/Cargo.toml
+++ b/common/meta/grpc/Cargo.toml
@@ -26,11 +26,16 @@ common-tracing = { path = "../../tracing" }
 
 derive_more = "0.99.17"
 futures = "0.3.21"
+once_cell = "1.10.0"
 prost = "=0.9.0"
 rand = "0.8.5"
+semver = "1.0.9"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.79"
 thiserror = "1.0.30"
 tonic = { version = "=0.6.2", features = ["transport", "codegen", "prost", "tls-roots", "tls"] }
 
 [dev-dependencies]
+
+[build-dependencies]
+common-building = { path = "../../building" }
diff --git a/common/meta/grpc/build.rs b/common/meta/grpc/build.rs
new file mode 100644
index 0000000000000..8f85cdb8bc8f1
--- /dev/null
+++ b/common/meta/grpc/build.rs
@@ -0,0 +1,16 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+fn main() {
+    common_building::setup();
+}
diff --git a/common/meta/grpc/src/grpc_client.rs b/common/meta/grpc/src/grpc_client.rs
index 716c886539f98..f7013109e19e2 100644
--- a/common/meta/grpc/src/grpc_client.rs
+++ b/common/meta/grpc/src/grpc_client.rs
@@ -43,6 +43,7 @@ use common_meta_types::protobuf::RaftRequest;
 use common_meta_types::protobuf::WatchRequest;
 use common_meta_types::protobuf::WatchResponse;
 use common_meta_types::ConnectionError;
+use common_meta_types::InvalidArgument;
 use common_meta_types::MetaError;
 use common_meta_types::MetaNetworkError;
 use common_meta_types::MetaResultError;
@@ -52,6 +53,7 @@ use common_tracing::tracing;
 use futures::stream::StreamExt;
 use prost::Message;
 use rand::Rng;
+use semver::Version;
 use serde::de::DeserializeOwned;
 use tonic::async_trait;
 use tonic::client::GrpcService;
@@ -63,10 +65,14 @@ use tonic::Code;
 use tonic::Request;
 use tonic::Status;
 
+use crate::from_digit_ver;
 use crate::grpc_action::MetaGrpcReadReq;
 use crate::grpc_action::MetaGrpcWriteReq;
 use crate::grpc_action::RequestFor;
 use crate::message;
+use crate::to_digit_ver;
+use crate::METACLI_COMMIT_SEMVER;
+use crate::MIN_METASRV_SEMVER;
 
 const AUTH_TOKEN_KEY: &str = "auth-token-bin";
 
@@ -200,9 +206,7 @@ impl MetaGrpcClient {
     ///
     /// The worker is a singleton and the returned handle is cheap to clone.
     /// When all handles are dropped the worker will quit, then the runtime will be destroyed.
-    pub async fn try_new(
-        conf: &RpcClientConf,
-    ) -> std::result::Result<Arc<ClientHandle>, ErrorCode> {
+    pub fn try_new(conf: &RpcClientConf) -> std::result::Result<Arc<ClientHandle>, ErrorCode> {
         Self::try_create(
             conf.get_endpoints(),
             &conf.username,
@@ -210,17 +214,18 @@ impl MetaGrpcClient {
             conf.timeout,
             conf.tls_conf.clone(),
         )
-        .await
     }
 
     #[tracing::instrument(level = "debug", skip(password))]
-    pub async fn try_create(
+    pub fn try_create(
         endpoints: Vec<String>,
         username: &str,
         password: &str,
         timeout: Option<Duration>,
         conf: Option<RpcClientTlsConfig>,
     ) -> Result<Arc<ClientHandle>> {
+        Self::endpoints_non_empty(&endpoints)?;
+
         let mgr = MetaChannelManager {
             timeout,
             conf: conf.clone(),
@@ -238,13 +243,12 @@ impl MetaGrpcClient {
 
         let worker = Arc::new(Self {
             conn_pool: Pool::new(mgr, Duration::from_millis(50)),
-            endpoints: RwLock::new(vec![]),
+            endpoints: RwLock::new(endpoints),
             username: username.to_string(),
             password: password.to_string(),
             token: RwLock::new(None),
             rt: rt.clone(),
         });
-        worker.set_endpoints(endpoints).await?;
 
         rt.spawn(Self::worker_loop(worker, rx));
 
@@ -379,8 +383,14 @@ impl MetaGrpcClient {
         let token = match t.clone() {
             Some(t) => t,
             None => {
-                let new_token =
-                    MetaGrpcClient::handshake(&mut client, &self.username, &self.password).await?;
+                let new_token = Self::handshake(
+                    &mut client,
+                    &METACLI_COMMIT_SEMVER,
+                    &MIN_METASRV_SEMVER,
+                    &self.username,
+                    &self.password,
+                )
+                .await?;
                 *t = Some(new_token.clone());
                 new_token
             }
@@ -391,14 +401,19 @@ impl MetaGrpcClient {
         Ok(client)
     }
 
+    pub fn endpoints_non_empty(endpoints: &[String]) -> std::result::Result<(), MetaError> {
+        if endpoints.is_empty() {
+            return Err(MetaError::InvalidConfig("endpoints is empty".to_string()));
+        }
+        Ok(())
+    }
+
     #[tracing::instrument(level = "debug", skip(self))]
     pub async fn set_endpoints(
         &self,
         endpoints: Vec<String>,
     ) -> std::result::Result<(), MetaError> {
-        if endpoints.is_empty() {
-            return Err(MetaError::InvalidConfig("endpoints is empty".to_string()));
-        }
+        Self::endpoints_non_empty(&endpoints)?;
 
         let mut eps = self.endpoints.write().await;
         *eps = endpoints;
@@ -406,9 +421,46 @@ impl MetaGrpcClient {
     }
 
     /// Handshake with metasrv.
+    ///
+    /// - Check whether the versions of this client(`C`) and the remote metasrv(`S`) are compatible.
+    /// - Authorize this client.
+    ///
+    /// ## Check compatibility
+    ///
+    /// Both client `C` and  server `S` maintains two semantic-version:
+    /// - `C` maintains the its own semver(`C.ver`) and the minimal compatible `S` semver(`C.min_srv_ver`).
+    /// - `S` maintains the its own semver(`S.ver`) and the minimal compatible `S` semver(`S.min_cli_ver`).
+    ///
+    /// When handshaking:
+    /// - `C` sends its ver `C.ver` to `S`,
+    /// - When `S` receives handshake request, `S` asserts that `C.ver >= S.min_cli_ver`.
+    /// - Then `S` replies handshake-reply with its `S.ver`.
+    /// - When `C` receives the reply, `C` asserts that `S.ver >= C.min_srv_ver`.
+    ///
+    /// Handshake succeeds if both of these two assertions hold.
+    ///
+    /// E.g.:
+    /// - `S: (ver=3, min_cli_ver=1)` is compatible with `C: (ver=3, min_srv_ver=2)`.
+    /// - `S: (ver=4, min_cli_ver=4)` is **NOT** compatible with `C: (ver=3, min_srv_ver=2)`.
+    ///   Because although `S.ver(4) >= C.min_srv_ver(3)` holds,
+    ///   but `C.ver(3) >= S.min_cli_ver(4)` does not hold.
+    ///
+    /// ```text
+    /// C.ver:    1             3      4
+    /// C --------+-------------+------+------------>
+    ///           ^      .------'      ^
+    ///           |      |             |
+    ///           '-------------.      |
+    ///                  |      |      |
+    ///                  v      |      |
+    /// S ---------------+------+------+------------>
+    /// S.ver:           2      3      4
+    /// ```
     #[tracing::instrument(level = "debug", skip(client, password))]
-    async fn handshake(
+    pub async fn handshake(
         client: &mut MetaServiceClient<Channel>,
+        client_ver: &Version,
+        min_metasrv_ver: &Version,
         username: &str,
         password: &str,
     ) -> std::result::Result<Vec<u8>, MetaError> {
@@ -419,9 +471,10 @@ impl MetaGrpcClient {
         let mut payload = vec![];
         auth.encode(&mut payload)?;
 
-        let req = Request::new(futures::stream::once(async {
+        let my_ver = to_digit_ver(client_ver);
+        let req = Request::new(futures::stream::once(async move {
             HandshakeRequest {
-                protocol_version: 0,
+                protocol_version: my_ver,
                 payload,
             }
         }));
@@ -429,7 +482,33 @@ impl MetaGrpcClient {
         let rx = client.handshake(req).await?;
         let mut rx = rx.into_inner();
 
-        let resp = rx.next().await.expect("Must respond from handshake")?;
+        let res = rx.next().await.ok_or_else(|| {
+            MetaNetworkError::ConnectionError(ConnectionError::new(
+                AnyError::error("handshake returns nothing"),
+                "",
+            ))
+        })?;
+
+        let resp = res?;
+
+        // backward compatibility: no version in handshake.
+        // TODO(xp): remove this when merged.
+        if resp.protocol_version > 0 {
+            let min_compatible = to_digit_ver(min_metasrv_ver);
+            if resp.protocol_version < min_compatible {
+                return Err(MetaError::MetaNetworkError(
+                    MetaNetworkError::InvalidArgument(InvalidArgument::new(
+                        AnyError::error(format!(
+                            "metasrv protocol_version({}) < meta-client min-compatible({})",
+                            from_digit_ver(resp.protocol_version),
+                            min_metasrv_ver,
+                        )),
+                        "",
+                    )),
+                ));
+            }
+        }
+
         let token = resp.payload;
         Ok(token)
     }
diff --git a/common/meta/grpc/src/lib.rs b/common/meta/grpc/src/lib.rs
index d5faa6c29c18e..34730eead7268 100644
--- a/common/meta/grpc/src/lib.rs
+++ b/common/meta/grpc/src/lib.rs
@@ -23,3 +23,37 @@ pub use grpc_action::RequestFor;
 pub use grpc_client::ClientHandle;
 pub use grpc_client::MetaGrpcClient;
 pub use message::ClientWorkerRequest;
+use once_cell::sync::Lazy;
+use semver::BuildMetadata;
+use semver::Prerelease;
+use semver::Version;
+
+pub static METACLI_COMMIT_SEMVER: Lazy<Version> = Lazy::new(|| {
+    let build_semver = option_env!("VERGEN_GIT_SEMVER");
+    let semver = build_semver.expect("VERGEN_GIT_SEMVER can not be None");
+
+    let semver = if semver.starts_with("v") {
+        &semver[1..]
+    } else {
+        semver
+    };
+
+    Version::parse(semver).unwrap()
+});
+
+/// Oldest compatible nightly metasrv version
+pub static MIN_METASRV_SEMVER: Version = Version {
+    major: 0,
+    minor: 7,
+    patch: 59,
+    pre: Prerelease::EMPTY,
+    build: BuildMetadata::EMPTY,
+};
+
+pub fn to_digit_ver(v: &Version) -> u64 {
+    v.major * 1_000_000 + v.minor * 1_000 + v.patch
+}
+
+pub fn from_digit_ver(u: u64) -> Version {
+    Version::new(u / 1_000_000, u / 1_000 % 1_000, u % 1_000)
+}
diff --git a/common/meta/grpc/tests/it/grpc_client.rs b/common/meta/grpc/tests/it/grpc_client.rs
index 10c1cbcabef10..44d87100845c5 100644
--- a/common/meta/grpc/tests/it/grpc_client.rs
+++ b/common/meta/grpc/tests/it/grpc_client.rs
@@ -31,9 +31,7 @@ async fn test_grpc_client_action_timeout() {
     // server's handshake impl will sleep 2secs.
     let timeout = Duration::from_secs(3);
 
-    let client = MetaGrpcClient::try_create(vec![srv_addr], "", "", Some(timeout), None)
-        .await
-        .unwrap();
+    let client = MetaGrpcClient::try_create(vec![srv_addr], "", "", Some(timeout), None).unwrap();
 
     let res = client
         .get_database(GetDatabaseReq::new("tenant1", "xx"))
@@ -49,9 +47,7 @@ async fn test_grpc_client_handshake_timeout() {
     let srv_addr = start_grpc_server();
 
     let timeout = Duration::from_secs(1);
-    let res = MetaGrpcClient::try_create(vec![srv_addr], "", "", Some(timeout), None)
-        .await
-        .unwrap();
+    let res = MetaGrpcClient::try_create(vec![srv_addr], "", "", Some(timeout), None).unwrap();
 
     let client = res.make_client().await;
     let got = client.unwrap_err();
diff --git a/common/meta/raft-store/src/config.rs b/common/meta/raft-store/src/config.rs
index 7355b6205dace..7b5d2a0f8aa1c 100644
--- a/common/meta/raft-store/src/config.rs
+++ b/common/meta/raft-store/src/config.rs
@@ -40,7 +40,7 @@ pub static DATABEND_COMMIT_VERSION: Lazy<String> = Lazy::new(|| {
     ver
 });
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, serde::Serialize)]
 pub struct RaftConfig {
     /// Identify a config.
     /// This is only meant to make debugging easier with more than one Config involved.
diff --git a/common/meta/types/src/lib.rs b/common/meta/types/src/lib.rs
index 870ad14e4f411..f62dd86d133d2 100644
--- a/common/meta/types/src/lib.rs
+++ b/common/meta/types/src/lib.rs
@@ -113,6 +113,7 @@ pub use meta_errors::MetaError;
 pub use meta_errors::MetaResult;
 pub use meta_errors_into::ToMetaError;
 pub use meta_network_errors::ConnectionError;
+pub use meta_network_errors::InvalidArgument;
 pub use meta_network_errors::MetaNetworkError;
 pub use meta_network_errors::MetaNetworkResult;
 pub use meta_raft_errors::ForwardToLeader;
diff --git a/common/meta/types/src/meta_errors_into.rs b/common/meta/types/src/meta_errors_into.rs
index 4317a4a380754..de4d8791ae971 100644
--- a/common/meta/types/src/meta_errors_into.rs
+++ b/common/meta/types/src/meta_errors_into.rs
@@ -17,7 +17,9 @@ use std::fmt::Display;
 use anyerror::AnyError;
 use common_exception::ErrorCode;
 use prost::EncodeError;
+use tonic::Code;
 
+use crate::meta_network_errors::InvalidArgument;
 use crate::ConnectionError;
 use crate::MetaError;
 use crate::MetaNetworkError;
@@ -97,9 +99,30 @@ where E: Display + Send + Sync + 'static
 // ser/de to/from tonic::Status
 impl From<tonic::Status> for MetaError {
     fn from(status: tonic::Status) -> Self {
-        MetaError::MetaNetworkError(MetaNetworkError::ConnectionError(ConnectionError::new(
-            status, "",
-        )))
+        match status.code() {
+            Code::InvalidArgument => MetaError::MetaNetworkError(
+                MetaNetworkError::InvalidArgument(InvalidArgument::new(status, "")),
+            ),
+            // Code::Ok => {}
+            // Code::Cancelled => {}
+            // Code::Unknown => {}
+            // Code::DeadlineExceeded => {}
+            // Code::NotFound => {}
+            // Code::AlreadyExists => {}
+            // Code::PermissionDenied => {}
+            // Code::ResourceExhausted => {}
+            // Code::FailedPrecondition => {}
+            // Code::Aborted => {}
+            // Code::OutOfRange => {}
+            // Code::Unimplemented => {}
+            // Code::Internal => {}
+            // Code::Unavailable => {}
+            // Code::DataLoss => {}
+            // Code::Unauthenticated => {}
+            _ => MetaError::MetaNetworkError(MetaNetworkError::ConnectionError(
+                ConnectionError::new(status, ""),
+            )),
+        }
     }
 }
 
diff --git a/common/meta/types/src/meta_network_errors.rs b/common/meta/types/src/meta_network_errors.rs
index 019b560175ab5..4c5bd256d7346 100644
--- a/common/meta/types/src/meta_network_errors.rs
+++ b/common/meta/types/src/meta_network_errors.rs
@@ -35,6 +35,9 @@ pub enum MetaNetworkError {
 
     #[error(transparent)]
     BadAddressFormat(AnyError),
+
+    #[error(transparent)]
+    InvalidArgument(#[from] InvalidArgument),
 }
 
 impl From<MetaNetworkError> for ErrorCode {
@@ -53,6 +56,9 @@ impl From<MetaNetworkError> for ErrorCode {
                 ErrorCode::TLSConfigurationFailure(any_err.to_string())
             }
             MetaNetworkError::DnsParseError(_) => ErrorCode::DnsParseError(net_err.to_string()),
+            MetaNetworkError::InvalidArgument(inv_arg) => {
+                ErrorCode::InvalidArgument(inv_arg.to_string())
+            }
         }
     }
 }
@@ -76,6 +82,23 @@ impl ConnectionError {
     }
 }
 
+#[derive(Error, Serialize, Deserialize, Debug, Clone, PartialEq)]
+#[error("InvalidArgument: {msg} source: {source}")]
+pub struct InvalidArgument {
+    msg: String,
+    #[source]
+    source: AnyError,
+}
+
+impl InvalidArgument {
+    pub fn new(source: impl std::error::Error + 'static, msg: impl Into<String>) -> Self {
+        Self {
+            msg: msg.into(),
+            source: AnyError::new(&source),
+        }
+    }
+}
+
 impl From<std::net::AddrParseError> for MetaNetworkError {
     fn from(error: std::net::AddrParseError) -> Self {
         MetaNetworkError::BadAddressFormat(AnyError::new(&error))
diff --git a/docs/doc/50-manage/60-upgrade/10-compatibility.md b/docs/doc/50-manage/60-upgrade/10-compatibility.md
new file mode 100644
index 0000000000000..9c2c7af14acaf
--- /dev/null
+++ b/docs/doc/50-manage/60-upgrade/10-compatibility.md
@@ -0,0 +1,88 @@
+---
+title: Query-Meta Compatibility
+sidebar_label: Query-Meta Compatibility
+description:
+  Investigate and manage the compatibility between databend-query and databend-meta
+---
+
+This guideline will introduce how to investigate and manage the compatibility between databend-query and databend-meta.
+
+## Find out the versions
+
+- To find out the build version of databend-query and its compatible databend-meta version:
+
+  ```shell
+  databend-query --cmd ver
+
+  # output:
+  version: 0.7.61-nightly
+  min-compatible-metasrv-version: 0.7.59
+  ```
+
+  Which means this build of databend-query(`0.7.61-nightly`) can talk to a databend-meta of at least version `0.7.59`, inclusive.
+
+- To find out the build version of databend-meta and its compatible databend-query version:
+
+  ```shell
+  databend-meta --cmd ver
+
+  # output:
+  version: 0.7.61-nightly
+  min-compatible-client-version: 0.7.57
+  ```
+
+  Which means this build of databend-meta(`0.7.61-nightly`) can talk to a databend-query of at least version `0.7.57`, inclusive.
+
+## Deploy compatible versions of databend-query and databend-meta
+
+A databend cluster has to be deployed with compatible versions of databend-query and databend-meta.
+A databend-query and databend-meta are compatible iff the following statements hold:
+
+```
+databend-query.version >= databend-meta.min-compatible-client-version
+databend-bend.version  >= databend-query.min-compatible-metasrv-version
+```
+
+:::caution
+
+If incompatible versions are deployed, an error `InvalidArgument` will occur when databend-query tries to connect to databend-meta,
+which can be found in databend-query log.
+Then databend-query will stop working.
+
+:::
+
+### How compatibility is checked
+
+Compatibility will be checked when a connection is established between meta-client(databend-query) and databend-meta, in a `handshake` RPC.
+
+The client `C`(databend-query) and the server `S`(databend-meta) maintains two semantic-version:
+
+- `C` maintains the its own semver(`C.ver`) and the minimal compatible `S` semver(`C.min_srv_ver`).
+- `S` maintains the its own semver(`S.ver`) and the minimal compatible `S` semver(`S.min_cli_ver`).
+
+When handshaking:
+
+- `C` sends its ver `C.ver` to `S`,
+- When `S` receives handshake request, `S` asserts that `C.ver >= S.min_cli_ver`.
+- Then `S` replies handshake-reply with its `S.ver`.
+- When `C` receives the reply, `C` asserts that `S.ver >= C.min_srv_ver`.
+
+Handshake succeeds if both of these two assertions hold.
+
+E.g.:
+- `S: (ver=3, min_cli_ver=1)` is compatible with `C: (ver=3, min_srv_ver=2)`.
+- `S: (ver=4, min_cli_ver=4)` is **NOT** compatible with `C: (ver=3, min_srv_ver=2)`.
+  Because although `S.ver(4) >= C.min_srv_ver(3)` holds,
+  but `C.ver(3) >= S.min_cli_ver(4)` does not hold.
+
+```text
+C.ver:    1             3      4
+C --------+-------------+------+------------>
+          ^      .------'      ^
+          |      |             |
+          '-------------.      |
+                 |      |      |
+                 v      |      |
+S ---------------+------+------+------------>
+S.ver:           2      3      4
+```
diff --git a/docs/doc/50-manage/60-upgrade/50-upgrade.md b/docs/doc/50-manage/60-upgrade/50-upgrade.md
new file mode 100644
index 0000000000000..dde245ddd3623
--- /dev/null
+++ b/docs/doc/50-manage/60-upgrade/50-upgrade.md
@@ -0,0 +1,87 @@
+---
+title: Upgrade
+sidebar_label: Upgrade query or meta
+description:
+  Upgrade databend-query or databend-meta without downtime
+---
+
+This guideline will introduce how to upgrade databend-query or databend-meta with new version.
+
+## Check compatibility
+
+Before upgrading, make sure the compatibility is still held:
+
+- To find out the build version of databend-query and its compatible databend-meta version:
+
+  ```shell
+  databend-query --cmd ver
+
+  # output:
+  version: 0.7.61-nightly
+  min-compatible-metasrv-version: 0.7.59
+  ```
+
+  Which means this build of databend-query(`0.7.61-nightly`) can talk to a databend-meta of at least version `0.7.59`, inclusive.
+
+- To find out the build version of databend-meta and its compatible databend-query version:
+
+  ```shell
+  databend-meta --cmd ver
+
+  # output:
+  version: 0.7.61-nightly
+  min-compatible-client-version: 0.7.57
+  ```
+
+  Which means this build of databend-meta(`0.7.61-nightly`) can talk to a databend-query of at least version `0.7.57`, inclusive.
+
+A databend-query and databend-meta are compatible iff the following statements hold:
+
+```
+databend-query.version >= databend-meta.min-compatible-client-version
+databend-bend.version  >= databend-query.min-compatible-metasrv-version
+```
+
+:::caution
+
+If incompatible versions are deployed, an error `InvalidArgument` will occur when databend-query tries to connect to databend-meta,
+which can be found in databend-query log.
+Then databend-query will stop working.
+
+:::
+
+## Upgrade
+
+- To Upgrade databend-query:
+
+  Upgrading databend-query is simple since it is a stateless service.
+  Just kill and re-start every node one by one:
+
+  ```shell
+  # Shutdown old binary
+  killall databend-query
+
+  # Bring up new binary
+  databend-query -c ...
+  ```
+
+  Then make sure everything goes well by checking the databend-query log.
+
+- To upgrade databend-meta:
+
+  Only upgrading databend-meta node one by one.
+
+  Since databend-meta is a stateful service and is consensus cluster,
+  taking too many databend-meta nodes offline(e.g., 3 offline nodes in a cluster of 5) affects the availability.
+
+  Kill and re-start every node one by one:
+
+  ```shell
+  # Shutdown old binary
+  killall databend-meta
+
+  # Bring up new binary
+  databend-meta -c ...
+  ```
+
+  Then make sure everything goes well by checking the databend-query log and databend-meta log.
diff --git a/docs/doc/50-manage/60-upgrade/_category_.json b/docs/doc/50-manage/60-upgrade/_category_.json
new file mode 100644
index 0000000000000..177c1fdb21a9f
--- /dev/null
+++ b/docs/doc/50-manage/60-upgrade/_category_.json
@@ -0,0 +1,7 @@
+{
+  "label": "Upgrade",
+  "link": {
+    "type": "generated-index",
+    "slug": "/operations/upgrade"
+  }
+}
diff --git a/metasrv/Cargo.toml b/metasrv/Cargo.toml
index 3abfa7fabacbe..2fcffd17800e0 100644
--- a/metasrv/Cargo.toml
+++ b/metasrv/Cargo.toml
@@ -51,6 +51,7 @@ once_cell = "1.10.0"
 poem = { version = "=1.3.16", features = ["rustls"] }
 prometheus = { version = "0.13.0", features = ["process"] }
 prost = "=0.9.0"
+semver = "1.0.9"
 serde = { version = "1.0.136", features = ["derive"] }
 serde-bridge = "0.0.3"
 serde_json = "1.0.79"
@@ -68,3 +69,6 @@ pretty_assertions = "1.2.1"
 regex = "1.5.5"
 reqwest = { version = "0.11.10", features = ["json"] }
 temp-env = "0.2.0"
+
+[build-dependencies]
+common-building = { path = "../common/building" }
diff --git a/metasrv/build.rs b/metasrv/build.rs
new file mode 100644
index 0000000000000..47059f3f03899
--- /dev/null
+++ b/metasrv/build.rs
@@ -0,0 +1,17 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+fn main() {
+    common_building::setup();
+}
diff --git a/metasrv/src/api/grpc/grpc_service.rs b/metasrv/src/api/grpc/grpc_service.rs
index b9c788879c559..753b5aa29efe4 100644
--- a/metasrv/src/api/grpc/grpc_service.rs
+++ b/metasrv/src/api/grpc/grpc_service.rs
@@ -49,6 +49,10 @@ use tonic::Streaming;
 use crate::executor::ActionHandler;
 use crate::meta_service::meta_service_impl::GrpcStream;
 use crate::meta_service::MetaNode;
+use crate::ver::from_digit_ver;
+use crate::ver::to_digit_ver;
+use crate::ver::METASRV_SEMVER;
+use crate::ver::MIN_METACLI_SEMVER;
 
 pub struct MetaServiceImpl {
     token: GrpcToken,
@@ -98,7 +102,21 @@ impl MetaService for MetaServiceImpl {
             protocol_version,
             payload,
         } = req;
-        let _ = protocol_version;
+
+        let min_compatible = to_digit_ver(&MIN_METACLI_SEMVER);
+
+        // backward compatibility: no version in handshake.
+        // TODO(xp): remove this when merged.
+        if protocol_version > 0 {
+            if protocol_version < min_compatible {
+                return Err(Status::invalid_argument(format!(
+                    "meta-client protocol_version({}) < metasrv min-compatible({})",
+                    from_digit_ver(protocol_version),
+                    MIN_METACLI_SEMVER,
+                )));
+            }
+        }
+
         let auth = BasicAuth::decode(&*payload).map_err(|e| Status::internal(e.to_string()))?;
 
         let user = "root";
@@ -112,8 +130,8 @@ impl MetaService for MetaServiceImpl {
                 .map_err(|e| Status::internal(e.to_string()))?;
 
             let resp = HandshakeResponse {
+                protocol_version: to_digit_ver(&METASRV_SEMVER),
                 payload: token.into_bytes(),
-                ..HandshakeResponse::default()
             };
             let output = futures::stream::once(async { Ok(resp) });
             Ok(Response::new(Box::pin(output)))
diff --git a/metasrv/src/bin/metasrv.rs b/metasrv/src/bin/metasrv.rs
index b535ab920fdeb..ebe770d5ffca3 100644
--- a/metasrv/src/bin/metasrv.rs
+++ b/metasrv/src/bin/metasrv.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::ops::Deref;
 use std::sync::Arc;
 
 use common_base::base::RuntimeTracker;
@@ -26,18 +27,28 @@ use databend_meta::api::HttpService;
 use databend_meta::configs::Config;
 use databend_meta::meta_service::MetaNode;
 use databend_meta::metrics::init_meta_metrics_recorder;
+use databend_meta::ver::METASRV_COMMIT_VERSION;
+use databend_meta::ver::METASRV_SEMVER;
+use databend_meta::ver::MIN_METACLI_SEMVER;
 
 #[databend_main]
 async fn main(_global_tracker: Arc<RuntimeTracker>) -> common_exception::Result<()> {
     let conf = Config::load()?;
 
+    if run_cmd(&conf) {
+        return Ok(());
+    }
+
     let _guards = init_global_tracing(
         "databend-meta",
         conf.log_dir.as_str(),
         conf.log_level.as_str(),
     );
 
-    tracing::info!("{:?}", conf.clone());
+    tracing::info!("Databend-meta version: {}", METASRV_COMMIT_VERSION.as_str());
+    tracing::info!("Config: {:?}", serde_json::to_string_pretty(&conf).unwrap());
+
+    conf.raft_config.check()?;
 
     init_sled_db(conf.raft_config.raft_dir.clone());
     init_meta_metrics_recorder();
@@ -80,3 +91,24 @@ async fn main(_global_tracker: Arc<RuntimeTracker>) -> common_exception::Result<
 
     Ok(())
 }
+
+fn run_cmd(conf: &Config) -> bool {
+    if conf.cmd.is_empty() {
+        return false;
+    }
+
+    match conf.cmd.as_str() {
+        "ver" => {
+            println!("version: {}", METASRV_SEMVER.deref());
+            println!("min-compatible-client-version: {}", MIN_METACLI_SEMVER);
+        }
+        _ => {
+            eprintln!("Invalid cmd: {}", conf.cmd);
+            eprintln!("Available cmds:");
+            eprintln!("  --cmd ver");
+            eprintln!("    Print version and min compatible meta-client version");
+        }
+    }
+
+    true
+}
diff --git a/metasrv/src/configs/inner.rs b/metasrv/src/configs/inner.rs
index f74957ab62888..c0c2a627a87e2 100644
--- a/metasrv/src/configs/inner.rs
+++ b/metasrv/src/configs/inner.rs
@@ -17,8 +17,9 @@ use common_meta_types::MetaResult;
 
 use super::outer_v0::Config as OuterV0Config;
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, serde::Serialize)]
 pub struct Config {
+    pub cmd: String,
     pub config_file: String,
     pub log_level: String,
     pub log_dir: String,
@@ -35,6 +36,7 @@ pub struct Config {
 impl Default for Config {
     fn default() -> Self {
         Self {
+            cmd: "".to_string(),
             config_file: "".to_string(),
             log_level: "INFO".to_string(),
             log_dir: "./.databend/logs".to_string(),
@@ -54,7 +56,7 @@ impl Config {
     ///
     /// In the future, we could have `ConfigV1` and `ConfigV2`.
     pub fn load() -> MetaResult<Self> {
-        let cfg = OuterV0Config::load()?.try_into()?;
+        let cfg = OuterV0Config::load()?.into();
 
         Ok(cfg)
     }
diff --git a/metasrv/src/configs/outer_v0.rs b/metasrv/src/configs/outer_v0.rs
index a11b0cd962e4e..b8feb77483db1 100644
--- a/metasrv/src/configs/outer_v0.rs
+++ b/metasrv/src/configs/outer_v0.rs
@@ -31,6 +31,13 @@ use super::inner::Config as InnerConfig;
 #[clap(about, version, author)]
 #[serde(default)]
 pub struct Config {
+    /// Run a command
+    ///
+    /// Supported commands:
+    /// - `ver`: print version and quit.
+    #[clap(long, default_value = "")]
+    pub cmd: String,
+
     #[clap(long, short = 'c', default_value = "")]
     pub config_file: String,
 
@@ -69,11 +76,10 @@ impl Default for Config {
     }
 }
 
-impl TryInto<InnerConfig> for Config {
-    type Error = MetaError;
-
-    fn try_into(self) -> MetaResult<InnerConfig> {
-        Ok(InnerConfig {
+impl Into<InnerConfig> for Config {
+    fn into(self) -> InnerConfig {
+        InnerConfig {
+            cmd: self.cmd,
             config_file: self.config_file,
             log_level: self.log_level,
             log_dir: self.log_dir,
@@ -83,14 +89,15 @@ impl TryInto<InnerConfig> for Config {
             grpc_api_address: self.grpc_api_address,
             grpc_tls_server_cert: self.grpc_tls_server_cert,
             grpc_tls_server_key: self.grpc_tls_server_key,
-            raft_config: self.raft_config.try_into()?,
-        })
+            raft_config: self.raft_config.into(),
+        }
     }
 }
 
 impl From<InnerConfig> for Config {
     fn from(inner: InnerConfig) -> Self {
         Self {
+            cmd: inner.cmd,
             config_file: inner.config_file,
             log_level: inner.log_level,
             log_dir: inner.log_dir,
@@ -235,6 +242,8 @@ impl Into<Config> for ConfigViaEnv {
         };
 
         Config {
+            // cmd should only be passed in from CLI
+            cmd: "".to_string(),
             config_file: self.metasrv_config_file,
             log_level: self.metasrv_log_level,
             log_dir: self.metasrv_log_dir,
@@ -331,11 +340,9 @@ impl Default for RaftConfig {
     }
 }
 
-impl TryInto<InnerRaftConfig> for RaftConfig {
-    type Error = MetaError;
-
-    fn try_into(self) -> MetaResult<InnerRaftConfig> {
-        let irc = InnerRaftConfig {
+impl Into<InnerRaftConfig> for RaftConfig {
+    fn into(self) -> InnerRaftConfig {
+        InnerRaftConfig {
             config_id: self.config_id,
             raft_listen_host: self.raft_listen_host,
             raft_advertise_host: self.raft_advertise_host,
@@ -350,11 +357,7 @@ impl TryInto<InnerRaftConfig> for RaftConfig {
             join: self.join,
             id: self.id,
             sled_tree_prefix: self.sled_tree_prefix,
-        };
-
-        irc.check()?;
-
-        Ok(irc)
+        }
     }
 }
 
diff --git a/metasrv/src/lib.rs b/metasrv/src/lib.rs
index bd0a30c22e84c..68d0d5cac6655 100644
--- a/metasrv/src/lib.rs
+++ b/metasrv/src/lib.rs
@@ -22,6 +22,7 @@ pub mod meta_service;
 pub mod metrics;
 pub mod network;
 pub mod store;
+pub mod ver;
 pub mod watcher;
 
 pub trait Opened {
diff --git a/metasrv/src/ver.rs b/metasrv/src/ver.rs
new file mode 100644
index 0000000000000..81e123a080918
--- /dev/null
+++ b/metasrv/src/ver.rs
@@ -0,0 +1,67 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use once_cell::sync::Lazy;
+use semver::BuildMetadata;
+use semver::Prerelease;
+use semver::Version;
+
+pub static METASRV_COMMIT_VERSION: Lazy<String> = Lazy::new(|| {
+    let build_semver = option_env!("VERGEN_GIT_SEMVER");
+    let git_sha = option_env!("VERGEN_GIT_SHA_SHORT");
+    let rustc_semver = option_env!("VERGEN_RUSTC_SEMVER");
+    let timestamp = option_env!("VERGEN_BUILD_TIMESTAMP");
+
+    let ver = match (build_semver, git_sha, rustc_semver, timestamp) {
+        #[cfg(not(feature = "simd"))]
+        (Some(v1), Some(v2), Some(v3), Some(v4)) => format!("{}-{}({}-{})", v1, v2, v3, v4),
+        #[cfg(feature = "simd")]
+        (Some(v1), Some(v2), Some(v3), Some(v4)) => {
+            format!("{}-{}-simd({}-{})", v1, v2, v3, v4)
+        }
+        _ => String::new(),
+    };
+    ver
+});
+
+pub static METASRV_SEMVER: Lazy<Version> = Lazy::new(|| {
+    let build_semver = option_env!("VERGEN_GIT_SEMVER");
+    let semver = build_semver.expect("VERGEN_GIT_SEMVER can not be None");
+
+    let semver = if semver.starts_with("v") {
+        &semver[1..]
+    } else {
+        semver
+    };
+
+    Version::parse(semver).expect(&format!("Invalid semver: {:?}", semver))
+});
+
+/// Oldest compatible nightly meta-client version
+pub static MIN_METACLI_SEMVER: Version = Version {
+    major: 0,
+    minor: 7,
+    patch: 57,
+    pre: Prerelease::EMPTY,
+    build: BuildMetadata::EMPTY,
+};
+
+pub fn to_digit_ver(v: &Version) -> u64 {
+    v.major * 1_000_000 + v.minor * 1_000 + v.patch
+}
+
+pub fn from_digit_ver(u: u64) -> Version {
+    println!("{}", u);
+    Version::new(u / 1_000_000, u / 1_000 % 1_000, u % 1_000)
+}
diff --git a/metasrv/tests/it/grpc/metasrv_grpc_api.rs b/metasrv/tests/it/grpc/metasrv_grpc_api.rs
index 591b2c320237c..bb21cf4182a9c 100644
--- a/metasrv/tests/it/grpc/metasrv_grpc_api.rs
+++ b/metasrv/tests/it/grpc/metasrv_grpc_api.rs
@@ -45,7 +45,7 @@ async fn test_restart() -> anyhow::Result<()> {
 
     let (mut tc, addr) = crate::tests::start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr.clone()], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr.clone()], "root", "xxx", None, None)?;
 
     tracing::info!("--- upsert kv");
     {
@@ -105,7 +105,7 @@ async fn test_restart() -> anyhow::Result<()> {
     tokio::time::sleep(Duration::from_millis(10_000)).await;
 
     // try to reconnect the restarted server.
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     tracing::info!("--- get kv");
     {
@@ -200,8 +200,8 @@ async fn test_join() -> anyhow::Result<()> {
     let addr0 = tc0.config.grpc_api_address.clone();
     let addr1 = tc1.config.grpc_api_address.clone();
 
-    let client0 = MetaGrpcClient::try_create(vec![addr0], "root", "xxx", None, None).await?;
-    let client1 = MetaGrpcClient::try_create(vec![addr1], "root", "xxx", None, None).await?;
+    let client0 = MetaGrpcClient::try_create(vec![addr0], "root", "xxx", None, None)?;
+    let client1 = MetaGrpcClient::try_create(vec![addr1], "root", "xxx", None, None)?;
 
     let clients = vec![client0, client1];
 
diff --git a/metasrv/tests/it/grpc/metasrv_grpc_export.rs b/metasrv/tests/it/grpc/metasrv_grpc_export.rs
index b4285bd4924c6..c9cc3887531e8 100644
--- a/metasrv/tests/it/grpc/metasrv_grpc_export.rs
+++ b/metasrv/tests/it/grpc/metasrv_grpc_export.rs
@@ -37,7 +37,7 @@ async fn test_export() -> anyhow::Result<()> {
     async {
         let (_tc, addr) = crate::tests::start_metasrv().await?;
 
-        let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+        let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
         tracing::info!("--- upsert kv");
         {
diff --git a/metasrv/tests/it/grpc/metasrv_grpc_handshake.rs b/metasrv/tests/it/grpc/metasrv_grpc_handshake.rs
new file mode 100644
index 0000000000000..f2020ac37b098
--- /dev/null
+++ b/metasrv/tests/it/grpc/metasrv_grpc_handshake.rs
@@ -0,0 +1,124 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Test metasrv SchemaApi by writing to one node and then reading from another,
+//! on a restarted cluster.
+
+use std::ops::Deref;
+use std::time::Duration;
+
+use common_base::base::tokio;
+use common_grpc::ConnectionFactory;
+use common_meta_grpc::from_digit_ver;
+use common_meta_grpc::to_digit_ver;
+use common_meta_grpc::MetaGrpcClient;
+use common_meta_grpc::METACLI_COMMIT_SEMVER;
+use common_meta_grpc::MIN_METASRV_SEMVER;
+use common_meta_types::protobuf::meta_service_client::MetaServiceClient;
+use common_tracing::tracing;
+use databend_meta::ver::MIN_METACLI_SEMVER;
+use semver::Version;
+
+use crate::init_meta_ut;
+use crate::tests::start_metasrv;
+
+/// - Test client version < serverside min-compatible-client-ver.
+/// - Test metasrv version < client min-compatible-metasrv-ver.
+#[tokio::test(flavor = "multi_thread", worker_threads = 3)]
+async fn test_metasrv_handshake() -> anyhow::Result<()> {
+    let (_log_guards, ut_span) = init_meta_ut!();
+    let _ent = ut_span.enter();
+
+    fn smaller_ver(v: &Version) -> Version {
+        if v.major > 0 {
+            Version::new(v.major - 1, v.minor, v.patch)
+        } else if v.minor > 0 {
+            Version::new(0, v.minor - 1, v.patch)
+        } else if v.patch > 0 {
+            Version::new(0, 0, v.patch - 1)
+        } else {
+            unreachable!("can not build a semver smaller than {:?}", v)
+        }
+    }
+
+    let (_tc, addr) = start_metasrv().await?;
+
+    let c = ConnectionFactory::create_rpc_channel(addr, Some(Duration::from_millis(1000)), None)
+        .await?;
+    let mut client = MetaServiceClient::new(c);
+
+    tracing::info!("--- client has smaller ver than S.min_cli_ver");
+    {
+        let min_client_ver = &MIN_METACLI_SEMVER;
+        let cli_ver = smaller_ver(min_client_ver);
+
+        let res =
+            MetaGrpcClient::handshake(&mut client, &cli_ver, &MIN_METASRV_SEMVER, "root", "xxx")
+                .await;
+
+        tracing::debug!("handshake res: {:?}", res);
+        let e = res.unwrap_err();
+
+        let want = format!(
+            "meta-client protocol_version({}) < metasrv min-compatible({})",
+            cli_ver, MIN_METACLI_SEMVER
+        );
+        assert!(e.to_string().contains(&want), "handshake err: {:?}", e);
+    }
+
+    tracing::info!("--- server has smaller ver than C.min_srv_ver");
+    {
+        let min_srv_ver = &MIN_METASRV_SEMVER;
+        let mut min_srv_ver = min_srv_ver.clone();
+        min_srv_ver.major += 1;
+
+        let res = MetaGrpcClient::handshake(
+            &mut client,
+            &METACLI_COMMIT_SEMVER,
+            &min_srv_ver,
+            "root",
+            "xxx",
+        )
+        .await;
+
+        tracing::debug!("handshake res: {:?}", res);
+        let e = res.unwrap_err();
+
+        let want = format!(
+            "metasrv protocol_version({}) < meta-client min-compatible({})",
+            // strip `nightly` from 0.7.57-nightly
+            from_digit_ver(to_digit_ver(METACLI_COMMIT_SEMVER.deref(),)),
+            min_srv_ver,
+        );
+        assert!(
+            e.to_string().contains(&want),
+            "handshake err: {:?} contains: {}",
+            e,
+            want
+        );
+    }
+
+    tracing::info!("--- old client using ver==0 is allowed");
+    {
+        let zero = Version::new(0, 0, 0);
+
+        let res =
+            MetaGrpcClient::handshake(&mut client, &zero, &MIN_METASRV_SEMVER, "root", "xxx").await;
+
+        tracing::debug!("handshake res: {:?}", res);
+        assert!(res.is_ok());
+    }
+
+    Ok(())
+}
diff --git a/metasrv/tests/it/grpc/metasrv_grpc_kv_api.rs b/metasrv/tests/it/grpc/metasrv_grpc_kv_api.rs
index c01919f9a3ca5..0cec1f42ac4ab 100644
--- a/metasrv/tests/it/grpc/metasrv_grpc_kv_api.rs
+++ b/metasrv/tests/it/grpc/metasrv_grpc_kv_api.rs
@@ -37,9 +37,7 @@ impl KVApiBuilder<Arc<ClientHandle>> for Builder {
     async fn build(&self) -> Arc<ClientHandle> {
         let (tc, addr) = start_metasrv().await.unwrap();
 
-        let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)
-            .await
-            .unwrap();
+        let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).unwrap();
 
         {
             let mut tcs = self.test_contexts.lock().unwrap();
diff --git a/metasrv/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs b/metasrv/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs
index 26c1ae227e2d4..c68bb2bf20641 100644
--- a/metasrv/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs
+++ b/metasrv/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs
@@ -189,8 +189,7 @@ async fn test_kv_api_restart_cluster_token_expired() -> anyhow::Result<()> {
         "xxx",
         None,
         None,
-    )
-    .await?;
+    )?;
 
     tracing::info!("--- test write on a fresh cluster");
     let key_suffix = "1st";
diff --git a/metasrv/tests/it/grpc/metasrv_grpc_schema_api.rs b/metasrv/tests/it/grpc/metasrv_grpc_schema_api.rs
index 2101d911f94a2..d330aa96aa256 100644
--- a/metasrv/tests/it/grpc/metasrv_grpc_schema_api.rs
+++ b/metasrv/tests/it/grpc/metasrv_grpc_schema_api.rs
@@ -28,7 +28,7 @@ async fn test_meta_grpc_client_database_create_get_drop() -> anyhow::Result<()>
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}
         .database_create_get_drop(client.as_ref())
@@ -42,7 +42,7 @@ async fn test_meta_grpc_client_database_create_get_drop_in_diff_tenant() -> anyh
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}
         .database_create_get_drop_in_diff_tenant(client.as_ref())
@@ -56,7 +56,7 @@ async fn test_meta_grpc_client_database_list() -> anyhow::Result<()> {
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}.database_list(client.as_ref()).await
 }
@@ -68,7 +68,7 @@ async fn test_meta_grpc_client_database_list_in_diff_tenant() -> anyhow::Result<
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}
         .database_list_in_diff_tenant(client.as_ref())
@@ -82,7 +82,7 @@ async fn test_meta_grpc_client_database_rename() -> anyhow::Result<()> {
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}.database_rename(&client).await
 }
@@ -94,7 +94,7 @@ async fn test_meta_grpc_client_database_drop_undrop_list_history() -> anyhow::Re
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}
         .database_drop_undrop_list_history(client.as_ref())
@@ -108,7 +108,7 @@ async fn test_meta_grpc_client_table_create_get_drop() -> anyhow::Result<()> {
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}
         .table_create_get_drop(client.as_ref())
@@ -122,7 +122,7 @@ async fn test_meta_grpc_client_table_rename() -> anyhow::Result<()> {
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}.table_rename(client.as_ref()).await
 }
@@ -134,7 +134,7 @@ async fn test_meta_grpc_client_table_upsert_option() -> anyhow::Result<()> {
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}
         .table_upsert_option(client.as_ref())
@@ -148,7 +148,7 @@ async fn test_meta_grpc_client_table_drop_undrop_list_history() -> anyhow::Resul
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}
         .table_drop_undrop_list_history(client.as_ref())
@@ -162,7 +162,7 @@ async fn test_meta_grpc_client_table_list() -> anyhow::Result<()> {
 
     let (_tc, addr) = start_metasrv().await?;
 
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     SchemaApiTestSuite {}.table_list(client.as_ref()).await
 }
diff --git a/metasrv/tests/it/grpc/metasrv_grpc_tls.rs b/metasrv/tests/it/grpc/metasrv_grpc_tls.rs
index 17373ca5ed38e..2c30c99fd011f 100644
--- a/metasrv/tests/it/grpc/metasrv_grpc_tls.rs
+++ b/metasrv/tests/it/grpc/metasrv_grpc_tls.rs
@@ -48,8 +48,7 @@ async fn test_tls_server() -> anyhow::Result<()> {
         domain_name: TEST_CN_NAME.to_string(),
     };
 
-    let client =
-        MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, Some(tls_conf)).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, Some(tls_conf))?;
 
     let r = client
         .get_table(("do not care", "do not care", "do not care").into())
@@ -91,7 +90,6 @@ async fn test_tls_client_config_failure() -> anyhow::Result<()> {
         None,
         Some(tls_conf),
     )
-    .await
     .unwrap();
 
     let c = r.get_kv("foo").await;
diff --git a/metasrv/tests/it/grpc/metasrv_grpc_watch.rs b/metasrv/tests/it/grpc/metasrv_grpc_watch.rs
index a12f78d51b72b..a9baf2e42bdb9 100644
--- a/metasrv/tests/it/grpc/metasrv_grpc_watch.rs
+++ b/metasrv/tests/it/grpc/metasrv_grpc_watch.rs
@@ -26,7 +26,7 @@ use common_meta_types::UpsertKVReq;
 use crate::init_meta_ut;
 
 async fn upsert_kv_client_main(addr: String, updates: Vec<UpsertKVReq>) -> anyhow::Result<()> {
-    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
 
     // update some kv
     for update in updates.iter() {
@@ -42,7 +42,7 @@ async fn test_watch_main(
     mut watch_events: Vec<Event>,
     updates: Vec<UpsertKVReq>,
 ) -> anyhow::Result<()> {
-    let client = MetaGrpcClient::try_create(vec![addr.clone()], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr.clone()], "root", "xxx", None, None)?;
 
     // let mut grpc_client = client.make_conn().await?;
 
diff --git a/metasrv/tests/it/grpc/mod.rs b/metasrv/tests/it/grpc/mod.rs
index 95b1c6be14b37..3be45e92cbcdd 100644
--- a/metasrv/tests/it/grpc/mod.rs
+++ b/metasrv/tests/it/grpc/mod.rs
@@ -14,6 +14,7 @@
 
 pub mod metasrv_grpc_api;
 mod metasrv_grpc_export;
+pub mod metasrv_grpc_handshake;
 pub mod metasrv_grpc_kv_api;
 pub mod metasrv_grpc_kv_api_restart_cluster;
 pub mod metasrv_grpc_schema_api;
diff --git a/metasrv/tests/it/tests/service.rs b/metasrv/tests/it/tests/service.rs
index 6c541aede3f5d..30e0c34433af5 100644
--- a/metasrv/tests/it/tests/service.rs
+++ b/metasrv/tests/it/tests/service.rs
@@ -148,7 +148,7 @@ impl MetaSrvTestContext {
     pub async fn grpc_client(&self) -> anyhow::Result<Arc<ClientHandle>> {
         let addr = self.config.grpc_api_address.clone();
 
-        let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None).await?;
+        let client = MetaGrpcClient::try_create(vec![addr], "root", "xxx", None, None)?;
         Ok(client)
     }
 
diff --git a/query/Cargo.toml b/query/Cargo.toml
index 5432e5111a8e6..205e9d14e1a84 100644
--- a/query/Cargo.toml
+++ b/query/Cargo.toml
@@ -98,6 +98,7 @@ rand = "0.8.5"
 regex = "1.5.5"
 reqwest = "0.11.10"
 rsa = "0.5.0"
+semver = "1.0.9"
 serde = { version = "1.0.136", features = ["derive"] }
 serde-bridge = "0.0.3"
 serde_json = "1.0.79"
diff --git a/query/bin/databend-query.rs b/query/bin/databend-query.rs
index c7bcecbc1b562..b48df35460739 100644
--- a/query/bin/databend-query.rs
+++ b/query/bin/databend-query.rs
@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::ops::Deref;
 use std::sync::Arc;
 
 use common_base::base::RuntimeTracker;
 use common_macros::databend_main;
 use common_meta_embedded::MetaEmbedded;
+use common_meta_grpc::MIN_METASRV_SEMVER;
 use common_metrics::init_default_metrics_recorder;
 use common_tracing::init_global_tracing;
 use common_tracing::set_panic_hook;
@@ -31,11 +33,16 @@ use databend_query::servers::Server;
 use databend_query::servers::ShutdownHandle;
 use databend_query::sessions::SessionManager;
 use databend_query::Config;
+use databend_query::QUERY_SEMVER;
 
 #[databend_main]
 async fn main(_global_tracker: Arc<RuntimeTracker>) -> common_exception::Result<()> {
     let conf: Config = Config::load()?;
 
+    if run_cmd(&conf) {
+        return Ok(());
+    }
+
     if conf.meta.address.is_empty() {
         MetaEmbedded::init_global_meta_store(conf.meta.embedded_dir.clone()).await?;
     }
@@ -153,3 +160,24 @@ async fn main(_global_tracker: Arc<RuntimeTracker>) -> common_exception::Result<
     tracing::info!("Shutdown server.");
     Ok(())
 }
+
+fn run_cmd(conf: &Config) -> bool {
+    if conf.cmd.is_empty() {
+        return false;
+    }
+
+    match conf.cmd.as_str() {
+        "ver" => {
+            println!("version: {}", QUERY_SEMVER.deref());
+            println!("min-compatible-metasrv-version: {}", MIN_METASRV_SEMVER);
+        }
+        _ => {
+            eprintln!("Invalid cmd: {}", conf.cmd);
+            eprintln!("Available cmds:");
+            eprintln!("  --cmd ver");
+            eprintln!("    Print version and the min compatible databend-meta version");
+        }
+    }
+
+    true
+}
diff --git a/query/src/common/meta/meta_store.rs b/query/src/common/meta/meta_store.rs
index bed0b564118ef..29787fc7972d9 100644
--- a/query/src/common/meta/meta_store.rs
+++ b/query/src/common/meta/meta_store.rs
@@ -111,7 +111,7 @@ impl MetaStoreProvider {
             Ok(MetaStore::L(meta_store))
         } else {
             tracing::info!(conf = debug(&self.rpc_conf), "use remote meta");
-            let client = MetaGrpcClient::try_new(&self.rpc_conf).await?;
+            let client = MetaGrpcClient::try_new(&self.rpc_conf)?;
             Ok(MetaStore::R(client))
         }
     }
diff --git a/query/src/config/inner.rs b/query/src/config/inner.rs
index b39624d6c8c78..c9e42797f5dd6 100644
--- a/query/src/config/inner.rs
+++ b/query/src/config/inner.rs
@@ -34,6 +34,7 @@ use super::outer_v0::Config as OuterV0Config;
 /// All function should implemented based on this Config.
 #[derive(Clone, Default, Debug, PartialEq)]
 pub struct Config {
+    pub cmd: String,
     pub config_file: String,
 
     // Query engine config.
diff --git a/query/src/config/outer_v0.rs b/query/src/config/outer_v0.rs
index 7a2463bf3b2c8..282be27f23d4b 100644
--- a/query/src/config/outer_v0.rs
+++ b/query/src/config/outer_v0.rs
@@ -55,6 +55,10 @@ use super::inner::QueryConfig as InnerQueryConfig;
 #[clap(about, version, author)]
 #[serde(default)]
 pub struct Config {
+    /// Run a command and quit
+    #[clap(long, default_value_t)]
+    pub cmd: String,
+
     #[clap(long, short = 'c', default_value_t)]
     pub config_file: String,
 
@@ -123,6 +127,7 @@ impl Config {
 impl From<InnerConfig> for Config {
     fn from(inner: InnerConfig) -> Self {
         Self {
+            cmd: inner.cmd,
             config_file: inner.config_file,
             query: inner.query.into(),
             log: inner.log.into(),
@@ -138,6 +143,7 @@ impl TryInto<InnerConfig> for Config {
 
     fn try_into(self) -> Result<InnerConfig> {
         Ok(InnerConfig {
+            cmd: self.cmd,
             config_file: self.config_file,
             query: self.query.try_into()?,
             log: self.log.try_into()?,
diff --git a/query/src/lib.rs b/query/src/lib.rs
index 28c041e2cd86f..41e13b4959d98 100644
--- a/query/src/lib.rs
+++ b/query/src/lib.rs
@@ -41,3 +41,4 @@ mod version;
 
 pub use config::Config;
 pub use version::DATABEND_COMMIT_VERSION;
+pub use version::QUERY_SEMVER;
diff --git a/query/src/version.rs b/query/src/version.rs
index 7f00590fd63aa..c0d46bf7c5128 100644
--- a/query/src/version.rs
+++ b/query/src/version.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use once_cell::sync::Lazy;
+use semver::Version;
 
 pub static DATABEND_COMMIT_VERSION: Lazy<String> = Lazy::new(|| {
     let git_tag = option_env!("VERGEN_GIT_SEMVER");
@@ -30,3 +31,17 @@ pub static DATABEND_COMMIT_VERSION: Lazy<String> = Lazy::new(|| {
     };
     ver
 });
+
+pub static QUERY_SEMVER: Lazy<Version> = Lazy::new(|| {
+    //
+    let build_semver = option_env!("VERGEN_GIT_SEMVER");
+    let semver = build_semver.expect("VERGEN_GIT_SEMVER can not be None");
+
+    let semver = if semver.starts_with("v") {
+        &semver[1..]
+    } else {
+        semver
+    };
+
+    Version::parse(semver).expect(&format!("Invalid semver: {:?}", semver))
+});
diff --git a/tests/compat/test-compat.sh b/tests/compat/test-compat.sh
index 2f38d4c81987b..4d7b4049b5b66 100755
--- a/tests/compat/test-compat.sh
+++ b/tests/compat/test-compat.sh
@@ -1,60 +1,177 @@
 #!/bin/sh
 
+set -o errexit
+SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
+echo " === SCRIPT_PATH: $SCRIPT_PATH"
+# go to work tree root
+cd "$SCRIPT_PATH/../../"
+pwd
+
 usage()
 {
-    echo "test query being compatible with old metasrv"
-    echo "Expect current release binaries are in ./current/."
-    echo "Expect     old release binaries are in ./old/."
-    echo "Usage: $0"
+    echo " === test latest query being compatible with minimal compatible metasrv"
+    echo " === test latest metasrv being compatible with minimal compatible query"
+    echo " === Expect ./bins/current contains current binaries"
+    echo " === Usage: $0"
 }
 
-SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
+binary_url()
+{
+    local ver="$1"
+    echo "https://github.com/datafuselabs/databend/releases/download/v${ver}-nightly/databend-v${ver}-nightly-x86_64-unknown-linux-gnu.tar.gz"
+}
 
-echo "SCRIPT_PATH: $SCRIPT_PATH"
+# output: 0.7.58
+# Without prefix `v` and `-nightly`
+find_min_query_ver()
+{
+    ./bins/current/databend-meta --cmd ver | grep min-compatible-client-version |  awk '{print $2}'
+}
 
-# go to work tree root
-cd "$SCRIPT_PATH/../../"
+find_min_metasrv_ver()
+{
+    ./bins/current/databend-query --cmd ver | grep min-compatible-metasrv-version |  awk '{print $2}'
+}
 
-pwd
+# download a specific version of databend, untar it to folder `./bins/$ver`
+# `ver` is semver without prefix `v` or `-nightly`
+download_binary()
+{
+    local ver="$1"
 
-mkdir -p ./target/debug/
-chmod +x ./current/*
-chmod +x ./old/*
+    local url="$(binary_url $ver)"
+    local fn="databend-$ver.tar.gz"
+
+    if [ -f ./bins/$ver/databend-meta ]; then
+        echo " === binaries exist: $(ls ./bins/$ver/*)"
+        chmod +x ./bins/$ver/*
+        return
+    fi
+
+    if [ -f "$fn" ]; then
+        echo " === tar file exists: $fn"
+    else
+        echo " === Download binary ver: $ver"
+        echo " === Download binary url: $url"
+
+    # wget -q "$url" -o "$fn"
+    curl -L "$url" > "$fn"
+    fi
+
+    mkdir -p ./bins/$ver
+    tar -xf "$fn" -C ./bins/$ver
+
+    chmod +x ./bins/$ver/*
+}
+
+
+# Download test suite for a specific version of query.
+download_test_suite()
+{
+    local ver="$1"
 
-cp ./old/databend-query    ./target/debug/
-cp ./current/databend-meta ./target/debug/
+    local path="tests/suites/0_stateless/05_ddl"
 
-./target/debug/databend-meta --version
-./target/debug/databend-query --version
+    echo " === Download test suites from $ver:$path"
 
-# Not all cases in the latest repo will pass
-echo "Run some specific SQL tests."
+    rm -rf shadow || echo "no shadow dir"
 
-echo "Starting standalone DatabendQuery(debug)"
+    git clone \
+        -b v$ver-nightly \
+        --depth 1  \
+        --filter=blob:none  \
+        --sparse \
+        "https://github.com/datafuselabs/databend" \
+        shadow
 
-killall databend-query
-killall databend-meta
-sleep 1
+    cd shadow
+    git sparse-checkout set $path
 
-for bin in databend-query databend-meta; do
-	if test -n "$(pgrep $bin)"; then
-		echo "The $bin is not killed. force killing."
-		killall -9 $bin
-	fi
-done
+    echo " === Done download test suites from $ver:$path"
 
-echo 'Start databend-meta...'
-nohup target/debug/databend-meta --single --log-level=DEBUG &
-echo "Waiting on databend-meta 10 seconds..."
-python3 scripts/ci/wait_tcp.py --timeout 5 --port 9191
+    ls $path
+}
+
+kill_proc()
+{
+    local name="$1"
+
+    echo " === Kill $name ..."
+
+    killall "$name" || {
+        echo " === no "$name" to kill"
+        return
+    }
+
+    sleep 1
+
+    if test -n "$(pgrep $name)"; then
+        echo " === The $name is not killed. force killing."
+        killall -9 $name || echo " === no $Name to killall-9"
+    fi
+
+    echo " === Done kill $name"
+}
+
+# Test specified version of query and meta
+run_test()
+{
+    local query_ver="$1"
+    local metasrv_ver="$2"
+
+    echo " === Test with query-$query_ver and metasrv-$metasrv_ver"
+
+    cp ./bins/$query_ver/databend-query  ./target/debug/
+    cp ./bins/$metasrv_ver/databend-meta ./target/debug/
 
-echo 'Start databend-query...'
-nohup target/debug/databend-query -c scripts/ci/deploy/config/databend-query-node-1.toml &
+    echo " === metasrv version:"
+    ./target/debug/databend-meta --cmd ver || echo " === no version yet"
 
-echo "Waiting on databend-query 10 seconds..."
-python3 scripts/ci/wait_tcp.py --timeout 5 --port 3307
+    echo " === query version:"
+    ./target/debug/databend-query --cmd ver || echo " === no version yet"
 
-cd tests
+    kill_proc databend-query
+    kill_proc databend-meta
+
+    echo " === Clean old meta dir"
+    rm -rf .databend/meta || echo " === no meta dir to rm"
+
+    rm nohup.out
+
+    export RUST_BACKTRACE=1
+
+    echo ' === Start databend-meta...'
+    nohup target/debug/databend-meta --log-level=DEBUG &
+    python3 scripts/ci/wait_tcp.py --timeout 5 --port 9191
+
+    echo ' === Start databend-query...'
+    nohup target/debug/databend-query -c scripts/ci/deploy/config/databend-query-node-1.toml &
+    python3 scripts/ci/wait_tcp.py --timeout 5 --port 3307
+
+    echo " === Starting metasrv related test: 05_ddl"
+    if [ "$query_ver" = "current" ]; then
+        ./tests/databend-test --suites tests/suites --mode 'standalone' --run-dir 0_stateless -- 05_
+    else
+         download_test_suite $query_ver
+        ./tests/databend-test --suites shadow/$query_ver/tests/suites --mode 'standalone' --run-dir 0_stateless -- 05_
+    fi
+}
+
+# -- main --
+
+old_query_ver=$(find_min_query_ver)
+old_metasrv_ver=$(find_min_metasrv_ver)
+
+echo " === min query ver: $old_query_ver"
+echo " === min metasrv ver: $old_metasrv_ver"
+
+download_binary "$old_metasrv_ver"
+download_binary "$old_query_ver"
+
+mkdir -p ./target/debug/
 
-echo "Starting metasrv related test: 05_ddl"
-./databend-test --mode 'standalone' --run-dir 0_stateless -- 05_
+# run_test current current \
+#     || echo "failed"
+run_test current $old_metasrv_ver \
+    || echo "failed"
+# run_test $old_query_ver current
diff --git a/tools/metactl/src/grpc.rs b/tools/metactl/src/grpc.rs
index 9056cc79ff603..72757bf530236 100644
--- a/tools/metactl/src/grpc.rs
+++ b/tools/metactl/src/grpc.rs
@@ -17,8 +17,7 @@ use common_meta_types::protobuf::Empty;
 use tokio_stream::StreamExt;
 
 pub async fn export_meta(addr: &str) -> anyhow::Result<()> {
-    let client =
-        MetaGrpcClient::try_create(vec![addr.to_string()], "root", "xxx", None, None).await?;
+    let client = MetaGrpcClient::try_create(vec![addr.to_string()], "root", "xxx", None, None)?;
 
     let mut grpc_client = client.make_client().await?;
 
diff --git a/tools/metactl/src/main.rs b/tools/metactl/src/main.rs
index aa5242a521261..49883816e1521 100644
--- a/tools/metactl/src/main.rs
+++ b/tools/metactl/src/main.rs
@@ -371,8 +371,7 @@ async fn bench_client_num_conn(conf: &Config) -> anyhow::Result<()> {
 
     loop {
         i += 1;
-        let client =
-            MetaGrpcClient::try_create(vec![addr.to_string()], "root", "xxx", None, None).await?;
+        let client = MetaGrpcClient::try_create(vec![addr.to_string()], "root", "xxx", None, None)?;
 
         let res = client.get_kv("foo").await;
         println!("{}-th: get_kv(foo): {:?}", i, res);