From 940e7aa00bf7554675f9e3dbf2d1b2228676d8ee Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Tue, 10 Dec 2019 16:46:19 +0800 Subject: [PATCH] support force update configs in console --- share/resources/gflags.json | 3 +- src/graph/ConfigExecutor.cpp | 3 +- src/interface/meta.thrift | 1 + src/meta/ClientBasedGflagsManager.cpp | 21 ++++++----- src/meta/ClientBasedGflagsManager.h | 6 ++- src/meta/GflagsManager.h | 3 +- src/meta/KVBasedGflagsManager.cpp | 5 ++- src/meta/KVBasedGflagsManager.h | 3 +- src/meta/client/MetaClient.cpp | 9 ++--- src/meta/client/MetaClient.h | 2 +- .../configMan/SetConfigProcessor.cpp | 37 ++++++++++--------- .../processors/configMan/SetConfigProcessor.h | 8 ++-- src/parser/AdminSentences.h | 8 +++- src/parser/parser.yy | 7 +++- src/parser/scanner.lex | 2 + 15 files changed, 69 insertions(+), 49 deletions(-) diff --git a/share/resources/gflags.json b/share/resources/gflags.json index 41ee84e036d..bdc74727879 100644 --- a/share/resources/gflags.json +++ b/share/resources/gflags.json @@ -10,7 +10,8 @@ "v", "heartbeat_interval_secs", "meta_client_retry_times", - "slow_op_threshhold_ms" + "slow_op_threshhold_ms", + "wal_ttl" ], "IGNORED": [ "logging", diff --git a/src/graph/ConfigExecutor.cpp b/src/graph/ConfigExecutor.cpp index 59706ecd930..ba84e5a0767 100644 --- a/src/graph/ConfigExecutor.cpp +++ b/src/graph/ConfigExecutor.cpp @@ -147,7 +147,8 @@ void ConfigExecutor::setVariables() { return; } - auto future = ectx()->gflagsManager()->setConfig(module, name, type, value); + bool isForce = sentence_->isForce(); + auto future = ectx()->gflagsManager()->setConfig(module, name, type, value, isForce); auto *runner = ectx()->rctx()->runner(); auto cb = [this] (auto && resp) { diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift index c8bc66406f7..4c010f5922a 100644 --- a/src/interface/meta.thrift +++ b/src/interface/meta.thrift @@ -521,6 +521,7 @@ struct GetConfigResp { struct SetConfigReq { 1: ConfigItem item, + 2: bool force, } struct ListConfigsReq { diff --git a/src/meta/ClientBasedGflagsManager.cpp b/src/meta/ClientBasedGflagsManager.cpp index b35f3e4b9f3..202d5842026 100644 --- a/src/meta/ClientBasedGflagsManager.cpp +++ b/src/meta/ClientBasedGflagsManager.cpp @@ -22,10 +22,11 @@ template folly::Future> ClientBasedGflagsManager::set(const cpp2::ConfigModule& module, const std::string& name, const cpp2::ConfigType& type, - const ValueType& value) { + const ValueType& value, + const bool isForce) { std::string valueStr; valueStr.append(reinterpret_cast(&value), sizeof(value)); - return metaClient_->setConfig(module, name, type, std::move(valueStr)); + return metaClient_->setConfig(module, name, type, std::move(valueStr), isForce); } template<> @@ -33,23 +34,25 @@ folly::Future> ClientBasedGflagsManager::set(const cpp2::ConfigModule& module, const std::string& name, const cpp2::ConfigType& type, - const std::string& value) { - return metaClient_->setConfig(module, name, type, value); + const std::string& value, + const bool isForce) { + return metaClient_->setConfig(module, name, type, value, isForce); } folly::Future> ClientBasedGflagsManager::setConfig(const cpp2::ConfigModule& module, const std::string& name, - const cpp2::ConfigType& type, const VariantType &value) { + const cpp2::ConfigType& type, const VariantType &value, + const bool isForce) { switch (type) { case cpp2::ConfigType::INT64: - return set(module, name, type, boost::get(value)); + return set(module, name, type, boost::get(value), isForce); case cpp2::ConfigType::DOUBLE: - return set(module, name, type, boost::get(value)); + return set(module, name, type, boost::get(value), isForce); case cpp2::ConfigType::BOOL: - return set(module, name, type, boost::get(value)); + return set(module, name, type, boost::get(value), isForce); case cpp2::ConfigType::STRING: case cpp2::ConfigType::NESTED: - return set(module, name, type, boost::get(value)); + return set(module, name, type, boost::get(value), isForce); default: return Status::Error("parse value type error"); } diff --git a/src/meta/ClientBasedGflagsManager.h b/src/meta/ClientBasedGflagsManager.h index f602fac67f0..3ffdb22c6e1 100644 --- a/src/meta/ClientBasedGflagsManager.h +++ b/src/meta/ClientBasedGflagsManager.h @@ -23,7 +23,8 @@ class ClientBasedGflagsManager : public GflagsManager { folly::Future> setConfig(const cpp2::ConfigModule& module, const std::string& name, const cpp2::ConfigType& type, - const VariantType& value) override; + const VariantType& value, + const bool isForce = false) override; folly::Future>> getConfig(const cpp2::ConfigModule& module, const std::string& name) override; @@ -36,7 +37,8 @@ class ClientBasedGflagsManager : public GflagsManager { private: template folly::Future> set(const cpp2::ConfigModule& module, const std::string& name, - const cpp2::ConfigType& type, const ValueType& value); + const cpp2::ConfigType& type, const ValueType& value, + const bool isForce); MetaClient *metaClient_{nullptr}; }; diff --git a/src/meta/GflagsManager.h b/src/meta/GflagsManager.h index 42c6a87ac67..aeff3b45860 100644 --- a/src/meta/GflagsManager.h +++ b/src/meta/GflagsManager.h @@ -21,7 +21,8 @@ class GflagsManager { virtual folly::Future> setConfig(const cpp2::ConfigModule& module, const std::string& name, const cpp2::ConfigType& type, - const VariantType& value) = 0; + const VariantType& value, + const bool isForce) = 0; virtual folly::Future>> getConfig(const cpp2::ConfigModule& module, const std::string& name) = 0; diff --git a/src/meta/KVBasedGflagsManager.cpp b/src/meta/KVBasedGflagsManager.cpp index fd367c47291..65f55bacc90 100644 --- a/src/meta/KVBasedGflagsManager.cpp +++ b/src/meta/KVBasedGflagsManager.cpp @@ -29,8 +29,9 @@ Status KVBasedGflagsManager::init() { folly::Future> KVBasedGflagsManager::setConfig(const cpp2::ConfigModule& module, const std::string& name, - const cpp2::ConfigType& type, const VariantType &value) { - UNUSED(module); UNUSED(name); UNUSED(type); UNUSED(value); + const cpp2::ConfigType& type, const VariantType &value, + const bool isForce) { + UNUSED(module); UNUSED(name); UNUSED(type); UNUSED(value); UNUSED(isForce); LOG(FATAL) << "Unimplement!"; return Status::NotSupported(); } diff --git a/src/meta/KVBasedGflagsManager.h b/src/meta/KVBasedGflagsManager.h index 9702b1595a7..02085933dca 100644 --- a/src/meta/KVBasedGflagsManager.h +++ b/src/meta/KVBasedGflagsManager.h @@ -24,7 +24,8 @@ class KVBasedGflagsManager : public GflagsManager { folly::Future> setConfig(const cpp2::ConfigModule& module, const std::string& name, const cpp2::ConfigType& type, - const VariantType& value) override; + const VariantType& value, + const bool isForce) override; folly::Future>> getConfig(const cpp2::ConfigModule& module, const std::string& name) override; diff --git a/src/meta/client/MetaClient.cpp b/src/meta/client/MetaClient.cpp index 0ecf885db2e..86da6cb6bff 100644 --- a/src/meta/client/MetaClient.cpp +++ b/src/meta/client/MetaClient.cpp @@ -1325,7 +1325,8 @@ MetaClient::getConfig(const cpp2::ConfigModule& module, const std::string& name) folly::Future> MetaClient::setConfig(const cpp2::ConfigModule& module, const std::string& name, - const cpp2::ConfigType& type, const std::string& value) { + const cpp2::ConfigType& type, const std::string& value, + const bool isForce) { if (!configReady_) { return Status::Error("Not ready!"); } @@ -1333,11 +1334,11 @@ MetaClient::setConfig(const cpp2::ConfigModule& module, const std::string& name, item.set_module(module); item.set_name(name); item.set_type(type); - item.set_mode(cpp2::ConfigMode::MUTABLE); item.set_value(value); cpp2::SetConfigReq req; req.set_item(item); + req.set_force(isForce); folly::Promise> promise; auto future = promise.getFuture(); getResponse(std::move(req), [] (auto client, auto request) { @@ -1459,10 +1460,6 @@ void MetaClient::addLoadCfgTask() { } void MetaClient::updateGflagsValue(const ConfigItem& item) { - if (item.mode_ != cpp2::ConfigMode::MUTABLE) { - return; - } - std::string metaValue; switch (item.type_) { case cpp2::ConfigType::INT64: diff --git a/src/meta/client/MetaClient.h b/src/meta/client/MetaClient.h index b367110af28..3e8fdf1d1d4 100644 --- a/src/meta/client/MetaClient.h +++ b/src/meta/client/MetaClient.h @@ -240,7 +240,7 @@ class MetaClient { folly::Future> setConfig(const cpp2::ConfigModule& module, const std::string& name, - const cpp2::ConfigType& type, const std::string& value); + const cpp2::ConfigType& type, const std::string& value, const bool isForce); folly::Future>> listConfigs(const cpp2::ConfigModule& module); diff --git a/src/meta/processors/configMan/SetConfigProcessor.cpp b/src/meta/processors/configMan/SetConfigProcessor.cpp index d9c5ddab9bc..9a70657e999 100644 --- a/src/meta/processors/configMan/SetConfigProcessor.cpp +++ b/src/meta/processors/configMan/SetConfigProcessor.cpp @@ -46,8 +46,8 @@ void SetConfigProcessor::process(const cpp2::SetConfigReq& req) { auto module = req.get_item().get_module(); auto name = req.get_item().get_name(); auto type = req.get_item().get_type(); - auto mode = req.get_item().get_mode(); auto value = req.get_item().get_value(); + auto isForce = req.get_force(); folly::SharedMutex::WriteHolder wHolder(LockUtils::configLock()); cpp2::ErrorCode code = cpp2::ErrorCode::SUCCEEDED; @@ -57,21 +57,21 @@ void SetConfigProcessor::process(const cpp2::SetConfigReq& req) { if (module != cpp2::ConfigModule::ALL) { // When we set config of a specified module, check if it exists. // If it exists and is mutable, update it. - code = setOneConfig(module, name, type, mode, value, data); + code = setOneConfig(module, name, type, value, isForce, data); if (code != cpp2::ErrorCode::SUCCEEDED) { break; } } else { // When we set config of all module, then try to set it of every module. - code = setOneConfig(cpp2::ConfigModule::GRAPH, name, type, mode, value, data); + code = setOneConfig(cpp2::ConfigModule::GRAPH, name, type, value, isForce, data); if (code != cpp2::ErrorCode::SUCCEEDED) { break; } - code = setOneConfig(cpp2::ConfigModule::META, name, type, mode, value, data); + code = setOneConfig(cpp2::ConfigModule::META, name, type, value, isForce, data); if (code != cpp2::ErrorCode::SUCCEEDED) { break; } - code = setOneConfig(cpp2::ConfigModule::STORAGE, name, type, mode, value, data); + code = setOneConfig(cpp2::ConfigModule::STORAGE, name, type, value, isForce, data); if (code != cpp2::ErrorCode::SUCCEEDED) { break; } @@ -80,20 +80,20 @@ void SetConfigProcessor::process(const cpp2::SetConfigReq& req) { // For those nested options like FLAGS_rocksdb_db_options, if any field has changed, // we update them and put it back if (module != cpp2::ConfigModule::ALL) { - code = setNestedConfig(module, name, type, mode, value, data); + code = setNestedConfig(module, name, type, value, data); if (code != cpp2::ErrorCode::SUCCEEDED) { break; } } else { - code = setNestedConfig(cpp2::ConfigModule::GRAPH, name, type, mode, value, data); + code = setNestedConfig(cpp2::ConfigModule::GRAPH, name, type, value, data); if (code != cpp2::ErrorCode::SUCCEEDED) { break; } - code = setNestedConfig(cpp2::ConfigModule::META, name, type, mode, value, data); + code = setNestedConfig(cpp2::ConfigModule::META, name, type, value, data); if (code != cpp2::ErrorCode::SUCCEEDED) { break; } - code = setNestedConfig(cpp2::ConfigModule::STORAGE, name, type, mode, value, data); + code = setNestedConfig(cpp2::ConfigModule::STORAGE, name, type, value, data); if (code != cpp2::ErrorCode::SUCCEEDED) { break; } @@ -113,8 +113,8 @@ void SetConfigProcessor::process(const cpp2::SetConfigReq& req) { cpp2::ErrorCode SetConfigProcessor::setOneConfig(const cpp2::ConfigModule& module, const std::string& name, const cpp2::ConfigType& type, - const cpp2::ConfigMode& mode, const std::string& value, + const bool isForce, std::vector& data) { std::string configKey = MetaServiceUtils::configKey(module, name); auto ret = doGet(std::move(configKey)); @@ -123,10 +123,11 @@ cpp2::ErrorCode SetConfigProcessor::setOneConfig(const cpp2::ConfigModule& modul } cpp2::ConfigItem item = MetaServiceUtils::parseConfigValue(ret.value()); - if (item.get_mode() == cpp2::ConfigMode::IMMUTABLE) { + cpp2::ConfigMode curMode = item.get_mode(); + if (curMode == cpp2::ConfigMode::IMMUTABLE && !isForce) { return cpp2::ErrorCode::E_CONFIG_IMMUTABLE; } - std::string configValue = MetaServiceUtils::configValue(type, mode, value); + std::string configValue = MetaServiceUtils::configValue(type, curMode, value); data.emplace_back(std::move(configKey), std::move(configValue)); return cpp2::ErrorCode::SUCCEEDED; } @@ -134,22 +135,22 @@ cpp2::ErrorCode SetConfigProcessor::setOneConfig(const cpp2::ConfigModule& modul cpp2::ErrorCode SetConfigProcessor::setNestedConfig(const cpp2::ConfigModule& module, const std::string& name, const cpp2::ConfigType& type, - const cpp2::ConfigMode& mode, const std::string& updateList, - std::vector& data) { + std::vector& data) { std::string configKey = MetaServiceUtils::configKey(module, name); auto ret = doGet(std::move(configKey)); if (!ret.ok()) { return cpp2::ErrorCode::E_NOT_FOUND; } - cpp2::ConfigItem current = MetaServiceUtils::parseConfigValue(ret.value()); - if (current.get_mode() == cpp2::ConfigMode::IMMUTABLE) { + cpp2::ConfigItem item = MetaServiceUtils::parseConfigValue(ret.value()); + cpp2::ConfigMode curMode = item.get_mode(); + if (curMode == cpp2::ConfigMode::IMMUTABLE) { return cpp2::ErrorCode::E_CONFIG_IMMUTABLE; } Configuration conf; - auto confRet = conf.parseFromString(current.get_value()); + auto confRet = conf.parseFromString(item.get_value()); CHECK(confRet.ok()); std::vector updateFields; @@ -173,7 +174,7 @@ cpp2::ErrorCode SetConfigProcessor::setNestedConfig(const cpp2::ConfigModule& mo } if (updated) { - std::string configValue = MetaServiceUtils::configValue(type, mode, conf.dumpToString()); + std::string configValue = MetaServiceUtils::configValue(type, curMode, conf.dumpToString()); data.emplace_back(std::move(configKey), std::move(configValue)); } return cpp2::ErrorCode::SUCCEEDED; diff --git a/src/meta/processors/configMan/SetConfigProcessor.h b/src/meta/processors/configMan/SetConfigProcessor.h index 5064aba1773..95044f1a580 100644 --- a/src/meta/processors/configMan/SetConfigProcessor.h +++ b/src/meta/processors/configMan/SetConfigProcessor.h @@ -21,12 +21,12 @@ class SetConfigProcessor : public BaseProcessor { void process(const cpp2::SetConfigReq& req); cpp2::ErrorCode setOneConfig(const cpp2::ConfigModule& module, const std::string& name, - const cpp2::ConfigType& type, const cpp2::ConfigMode& mode, - const std::string& value, std::vector& data); + const cpp2::ConfigType& type, const std::string& value, + const bool isForce, std::vector& data); cpp2::ErrorCode setNestedConfig(const cpp2::ConfigModule& module, const std::string& name, - const cpp2::ConfigType& type, const cpp2::ConfigMode& mode, - const std::string& value, std::vector& data); + const cpp2::ConfigType& type, const std::string& value, + std::vector& data); private: explicit SetConfigProcessor(kvstore::KVStore* kvstore) diff --git a/src/parser/AdminSentences.h b/src/parser/AdminSentences.h index dcb80845be0..84039259281 100644 --- a/src/parser/AdminSentences.h +++ b/src/parser/AdminSentences.h @@ -284,10 +284,11 @@ class ConfigSentence final : public Sentence { subType_ = std::move(subType); } - ConfigSentence(SubType subType, ConfigRowItem* item) { + ConfigSentence(SubType subType, ConfigRowItem* item, bool force = false) { kind_ = Kind::kConfig; subType_ = std::move(subType); configItem_.reset(item); + isForce_ = force; } std::string toString() const override; @@ -300,8 +301,13 @@ class ConfigSentence final : public Sentence { return configItem_.get(); } + bool isForce() { + return isForce_; + } + private: SubType subType_{SubType::kUnknown}; + bool isForce_{false}; std::unique_ptr configItem_; }; diff --git a/src/parser/parser.yy b/src/parser/parser.yy index d085a97ca4e..151f0129423 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -104,7 +104,7 @@ class GraphScanner; %token KW_IF KW_NOT KW_EXISTS KW_WITH KW_FIRSTNAME KW_LASTNAME KW_EMAIL KW_PHONE KW_USER KW_USERS %token KW_PASSWORD KW_CHANGE KW_ROLE KW_GOD KW_ADMIN KW_GUEST KW_GRANT KW_REVOKE KW_ON %token KW_ROLES KW_BY KW_DOWNLOAD KW_HDFS -%token KW_CONFIGS KW_GET KW_DECLARE KW_GRAPH KW_META KW_STORAGE +%token KW_CONFIGS KW_GET KW_DECLARE KW_GRAPH KW_META KW_STORAGE KW_FORCE %token KW_TTL_DURATION KW_TTL_COL KW_DEFAULT %token KW_ORDER KW_ASC KW_LIMIT KW_OFFSET KW_GROUP %token KW_COUNT KW_COUNT_DISTINCT KW_SUM KW_AVG KW_MAX KW_MIN KW_STD KW_BIT_AND KW_BIT_OR KW_BIT_XOR @@ -1683,9 +1683,12 @@ get_config_sentence ; set_config_sentence - : KW_UPDATE KW_CONFIGS set_config_item { + : KW_UPDATE KW_CONFIGS set_config_item { $$ = new ConfigSentence(ConfigSentence::SubType::kSet, $3); } + | KW_UPDATE KW_CONFIGS set_config_item KW_FORCE { + $$ = new ConfigSentence(ConfigSentence::SubType::kSet, $3, true); + } ; host_list diff --git a/src/parser/scanner.lex b/src/parser/scanner.lex index 06307330025..9c0a10f3de4 100644 --- a/src/parser/scanner.lex +++ b/src/parser/scanner.lex @@ -144,6 +144,7 @@ IS ([Ii][Ss]) NULL ([Nn][Uu][Ll][Ll]) SNAPSHOT ([Ss][Nn][Aa][Pp][Ss][Hh][Oo][Tt]) SNAPSHOTS ([Ss][Nn][Aa][Pp][Ss][Hh][Oo][Tt][Ss]) +FORCE ([Ff][Oo][Rr][Cc][Ee]) LABEL ([a-zA-Z][_a-zA-Z0-9]*) DEC ([0-9]) @@ -274,6 +275,7 @@ IP_OCTET ([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]) {NULL} { return TokenType::KW_NULL; } {SNAPSHOT} { return TokenType::KW_SNAPSHOT; } {SNAPSHOTS} { return TokenType::KW_SNAPSHOTS; } +{FORCE} { return TokenType::KW_FORCE; } "." { return TokenType::DOT; } "," { return TokenType::COMMA; }