From bef3fff7311582c84368669811ff52db448633da Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Mon, 4 Dec 2023 16:47:12 +0200 Subject: [PATCH] Add support for read-only mode in ZooKeeper This commit enables the read-only flag when connecting to the ZooKeeper server. This flag is enabled by sending one extra byte when connecting, and then receiving one extra byte during the first response. In addition to that, we modify createIfNotExists to not complain about attempting to alter a read-only ZooKeeper cluster if the node already exists. This makes ClickHouse more useful in the event of a loss of quorum, user credentials are still accessible, which makes it possible to connect to the cluster and run read queries. Any DDL or DML query on a Distributed database or ReplicatedMergeTree table will correctly fail, since it needs to write to ZooKeeper to execute the query. Any non-distributed query will be possible, which is ok since the query was never replicated in the first place, there is no loss of consistency. Fixes #53749 as it seems to be the only thing 3.9 enforced. --- src/Common/ZooKeeper/IKeeper.cpp | 5 +++-- src/Common/ZooKeeper/IKeeper.h | 4 +++- src/Common/ZooKeeper/ZooKeeper.cpp | 5 +++++ src/Common/ZooKeeper/ZooKeeperConstants.h | 1 + src/Common/ZooKeeper/ZooKeeperImpl.cpp | 10 ++++++++-- src/Interpreters/ZooKeeperLog.cpp | 2 ++ 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/Common/ZooKeeper/IKeeper.cpp b/src/Common/ZooKeeper/IKeeper.cpp index f2e4b3213268..6c47ea68b848 100644 --- a/src/Common/ZooKeeper/IKeeper.cpp +++ b/src/Common/ZooKeeper/IKeeper.cpp @@ -144,6 +144,7 @@ const char * errorMessage(Error code) case Error::ZCLOSING: return "ZooKeeper is closing"; case Error::ZNOTHING: return "(not error) no server responses to process"; case Error::ZSESSIONMOVED: return "Session moved to another server, so operation is ignored"; + case Error::ZNOTREADONLY: return "State-changing request is passed to read-only server"; } UNREACHABLE(); @@ -156,7 +157,8 @@ bool isHardwareError(Error zk_return_code) || zk_return_code == Error::ZSESSIONMOVED || zk_return_code == Error::ZCONNECTIONLOSS || zk_return_code == Error::ZMARSHALLINGERROR - || zk_return_code == Error::ZOPERATIONTIMEOUT; + || zk_return_code == Error::ZOPERATIONTIMEOUT + || zk_return_code == Error::ZNOTREADONLY; } bool isUserError(Error zk_return_code) @@ -196,4 +198,3 @@ void MultiResponse::removeRootPath(const String & root_path) } } - diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index 6d6f8afbaee3..80dee2b5c819 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -109,7 +109,8 @@ enum class Error : int32_t ZAUTHFAILED = -115, /// Client authentication failed ZCLOSING = -116, /// ZooKeeper is closing ZNOTHING = -117, /// (not error) no server responses to process - ZSESSIONMOVED = -118 /// Session moved to another server, so operation is ignored + ZSESSIONMOVED = -118, /// Session moved to another server, so operation is ignored + ZNOTREADONLY = -119, /// State-changing request is passed to read-only server }; /// Network errors and similar. You should reinitialize ZooKeeper session in case of these errors @@ -445,6 +446,7 @@ enum State CONNECTING = 1, ASSOCIATING = 2, CONNECTED = 3, + READONLY = 5, NOTCONNECTED = 999 }; diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 436a4e14f14b..8d18494e9648 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -323,6 +323,9 @@ Coordination::Error ZooKeeper::tryCreate(const std::string & path, const std::st { Coordination::Error code = createImpl(path, data, mode, path_created); + if (code == Coordination::Error::ZNOTREADONLY && exists(path)) + return Coordination::Error::ZNODEEXISTS; + if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE || code == Coordination::Error::ZNODEEXISTS || @@ -345,6 +348,8 @@ void ZooKeeper::createIfNotExists(const std::string & path, const std::string & if (code == Coordination::Error::ZOK || code == Coordination::Error::ZNODEEXISTS) return; + else if (code == Coordination::Error::ZNOTREADONLY && exists(path)) + return; else throw KeeperException::fromPath(code, path); } diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 1a868963b572..a5c1d21eda6a 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -51,6 +51,7 @@ static constexpr int32_t KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT = 42; static constexpr int32_t CLIENT_HANDSHAKE_LENGTH = 44; static constexpr int32_t CLIENT_HANDSHAKE_LENGTH_WITH_READONLY = 45; static constexpr int32_t SERVER_HANDSHAKE_LENGTH = 36; +static constexpr int32_t SERVER_HANDSHAKE_LENGTH_WITH_READONLY = 37; static constexpr int32_t PASSWORD_LENGTH = 16; /// ZooKeeper has 1 MB node size and serialization limit by default, diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 4335ea4655f1..9ec7208d3eb2 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1,3 +1,4 @@ +#include "Common/ZooKeeper/ZooKeeperConstants.h" #include #include @@ -552,12 +553,13 @@ void ZooKeeper::connect( void ZooKeeper::sendHandshake() { - int32_t handshake_length = 44; + int32_t handshake_length = 45; int64_t last_zxid_seen = 0; int32_t timeout = args.session_timeout_ms; int64_t previous_session_id = 0; /// We don't support session restore. So previous session_id is always zero. constexpr int32_t passwd_len = 16; std::array passwd {}; + bool read_only = true; write(handshake_length); if (use_compression) @@ -568,6 +570,7 @@ void ZooKeeper::sendHandshake() write(timeout); write(previous_session_id); write(passwd); + write(read_only); flushWriteBuffer(); } @@ -577,9 +580,10 @@ void ZooKeeper::receiveHandshake() int32_t protocol_version_read; int32_t timeout; std::array passwd; + bool read_only; read(handshake_length); - if (handshake_length != SERVER_HANDSHAKE_LENGTH) + if (handshake_length != SERVER_HANDSHAKE_LENGTH && handshake_length != SERVER_HANDSHAKE_LENGTH_WITH_READONLY) throw Exception(Error::ZMARSHALLINGERROR, "Unexpected handshake length received: {}", handshake_length); read(protocol_version_read); @@ -607,6 +611,8 @@ void ZooKeeper::receiveHandshake() read(session_id); read(passwd); + if (handshake_length == SERVER_HANDSHAKE_LENGTH_WITH_READONLY) + read(read_only); } diff --git a/src/Interpreters/ZooKeeperLog.cpp b/src/Interpreters/ZooKeeperLog.cpp index 5b25ff41d6b3..c7fcece72b12 100644 --- a/src/Interpreters/ZooKeeperLog.cpp +++ b/src/Interpreters/ZooKeeperLog.cpp @@ -51,6 +51,7 @@ DataTypePtr getCoordinationErrorCodesEnumType() {"ZCLOSING", static_cast(Coordination::Error::ZCLOSING)}, {"ZNOTHING", static_cast(Coordination::Error::ZNOTHING)}, {"ZSESSIONMOVED", static_cast(Coordination::Error::ZSESSIONMOVED)}, + {"ZNOTREADONLY", static_cast(Coordination::Error::ZNOTREADONLY)}, }); } @@ -113,6 +114,7 @@ NamesAndTypesList ZooKeeperLogElement::getNamesAndTypes() {"CONNECTING", static_cast(Coordination::State::CONNECTING)}, {"ASSOCIATING", static_cast(Coordination::State::ASSOCIATING)}, {"CONNECTED", static_cast(Coordination::State::CONNECTED)}, + {"READONLY", static_cast(Coordination::State::READONLY)}, {"NOTCONNECTED", static_cast(Coordination::State::NOTCONNECTED)}, });