diff --git a/be/src/common/config.h b/be/src/common/config.h
index 42edb26fa7a10f..06cb4cbf974891 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -22,6 +22,9 @@
 
 namespace doris {
 namespace config {
+    // Dir of custom config file
+    CONF_String(custom_config_dir, "${DORIS_HOME}/conf");
+
     // cluster id
     CONF_Int32(cluster_id, "-1");
     // port on which ImpalaInternalService is exported
diff --git a/be/src/common/configbase.cpp b/be/src/common/configbase.cpp
index 115c2db855e952..c46c34b2cd5e4d 100644
--- a/be/src/common/configbase.cpp
+++ b/be/src/common/configbase.cpp
@@ -30,6 +30,7 @@
 
 #include "common/status.h"
 #include "gutil/strings/substitute.h"
+#include "util/filesystem_util.h"
 
 namespace doris {
 namespace config {
@@ -37,7 +38,7 @@ namespace config {
 std::map<std::string, Register::Field>* Register::_s_field_map = nullptr;
 std::map<std::string, std::string>* full_conf_map = nullptr;
 
-Properties props;
+std::mutex custom_conf_lock;
 
 // trim string
 std::string& trim(std::string& s) {
@@ -171,7 +172,7 @@ bool strtox(const std::string& valstr, std::string& retval) {
 }
 
 // load conf file
-bool Properties::load(const char* filename) {
+bool Properties::load(const char* filename, bool must_exist) {
     // if filename is null, use the empty props
     if (filename == nullptr) {
         return true;
@@ -180,8 +181,11 @@ bool Properties::load(const char* filename) {
     // open the conf file
     std::ifstream input(filename);
     if (!input.is_open()) {
-        std::cerr << "config::load() failed to open the file:" << filename << std::endl;
-        return false;
+        if (must_exist) {
+            std::cerr << "config::load() failed to open the file:" << filename << std::endl;
+            return false;
+        }
+        return true;
     }
 
     // load properties
@@ -217,9 +221,19 @@ bool Properties::load(const char* filename) {
 }
 
 template <typename T>
-bool Properties::get(const char* key, const char* defstr, T& retval) const {
+bool Properties::get_or_default(const char* key, const char* defstr, T& retval) const {
     const auto& it = file_conf_map.find(std::string(key));
-    std::string valstr = it != file_conf_map.end() ? it->second : std::string(defstr);
+    std::string valstr;
+    if (it == file_conf_map.end()) {
+        if (defstr == nullptr) {
+            // Not found in conf map, and no default value need to be set, just return
+            return true;
+        } else {
+            valstr = std::string(defstr);
+        }
+    } else {
+        valstr = it->second;
+    }
     trim(valstr);
     if (!replaceenv(valstr)) {
         return false;
@@ -227,6 +241,34 @@ bool Properties::get(const char* key, const char* defstr, T& retval) const {
     return strtox(valstr, retval);
 }
 
+void Properties::set(const std::string& key, const std::string& val) {
+    file_conf_map.emplace(key, val);
+}
+
+bool Properties::dump(const std::string& conffile) {
+    std::vector<std::string> files = { conffile };
+    Status st = FileSystemUtil::remove_paths(files); 
+    if (!st.ok()) {
+        return false;
+    }
+    st = FileSystemUtil::create_file(conffile);
+    if (!st.ok()) {
+        return false;
+    }
+
+    std::ofstream out(conffile);
+    out << "# THIS IS AN AUTO GENERATED CONFIG FILE.\n";
+    out << "# You can modify this file manually, and the configurations in this file\n";
+    out << "# will overwrite the configurations in be.conf\n";
+    out << "\n";
+    
+    for (auto const& iter : file_conf_map) {
+        out << iter.first << " = " << iter.second << "\n";
+    }
+    out.close();
+    return true;
+}
+
 template <typename T>
 bool update(const std::string& value, T& retval) {
     std::string valstr(value);
@@ -249,24 +291,27 @@ std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
     return out;
 }
 
-#define SET_FIELD(FIELD, TYPE, FILL_CONFMAP)                                                       \
-    if (strcmp((FIELD).type, #TYPE) == 0) {                                                        \
-        if (!props.get((FIELD).name, (FIELD).defval, *reinterpret_cast<TYPE*>((FIELD).storage))) { \
-            std::cerr << "config field error: " << (FIELD).name << std::endl;                      \
-            return false;                                                                          \
-        }                                                                                          \
-        if (FILL_CONFMAP) {                                                                        \
-            std::ostringstream oss;                                                                \
-            oss << (*reinterpret_cast<TYPE*>((FIELD).storage));                                    \
-            (*full_conf_map)[(FIELD).name] = oss.str();                                            \
-        }                                                                                          \
-        continue;                                                                                  \
+#define SET_FIELD(FIELD, TYPE, FILL_CONFMAP, SET_TO_DEFAULT)                                                  \
+    if (strcmp((FIELD).type, #TYPE) == 0) {                                                                   \
+        if (!props.get_or_default(                                                                            \
+                (FIELD).name, ((SET_TO_DEFAULT) ? (FIELD).defval : nullptr),                                  \
+                *reinterpret_cast<TYPE*>((FIELD).storage))) {                                                 \
+            std::cerr << "config field error: " << (FIELD).name << std::endl;                                 \
+            return false;                                                                                     \
+        }                                                                                                     \
+        if (FILL_CONFMAP) {                                                                                   \
+            std::ostringstream oss;                                                                           \
+            oss << (*reinterpret_cast<TYPE*>((FIELD).storage));                                               \
+            (*full_conf_map)[(FIELD).name] = oss.str();                                                       \
+        }                                                                                                     \
+        continue;                                                                                             \
     }
 
 // init conf fields
-bool init(const char* filename, bool fillconfmap) {
+bool init(const char* conf_file, bool fillconfmap, bool must_exist, bool set_to_default) {
+    Properties props;
     // load properties file
-    if (!props.load(filename)) {
+    if (!props.load(conf_file, must_exist)) {
         return false;
     }
     // fill full_conf_map ?
@@ -276,24 +321,24 @@ bool init(const char* filename, bool fillconfmap) {
 
     // set conf fields
     for (const auto& it : *Register::_s_field_map) {
-        SET_FIELD(it.second, bool, fillconfmap);
-        SET_FIELD(it.second, int16_t, fillconfmap);
-        SET_FIELD(it.second, int32_t, fillconfmap);
-        SET_FIELD(it.second, int64_t, fillconfmap);
-        SET_FIELD(it.second, double, fillconfmap);
-        SET_FIELD(it.second, std::string, fillconfmap);
-        SET_FIELD(it.second, std::vector<bool>, fillconfmap);
-        SET_FIELD(it.second, std::vector<int16_t>, fillconfmap);
-        SET_FIELD(it.second, std::vector<int32_t>, fillconfmap);
-        SET_FIELD(it.second, std::vector<int64_t>, fillconfmap);
-        SET_FIELD(it.second, std::vector<double>, fillconfmap);
-        SET_FIELD(it.second, std::vector<std::string>, fillconfmap);
+        SET_FIELD(it.second, bool, fillconfmap, set_to_default);
+        SET_FIELD(it.second, int16_t, fillconfmap, set_to_default);
+        SET_FIELD(it.second, int32_t, fillconfmap, set_to_default);
+        SET_FIELD(it.second, int64_t, fillconfmap, set_to_default);
+        SET_FIELD(it.second, double, fillconfmap, set_to_default);
+        SET_FIELD(it.second, std::string, fillconfmap, set_to_default);
+        SET_FIELD(it.second, std::vector<bool>, fillconfmap, set_to_default);
+        SET_FIELD(it.second, std::vector<int16_t>, fillconfmap, set_to_default);
+        SET_FIELD(it.second, std::vector<int32_t>, fillconfmap, set_to_default);
+        SET_FIELD(it.second, std::vector<int64_t>, fillconfmap, set_to_default);
+        SET_FIELD(it.second, std::vector<double>, fillconfmap, set_to_default);
+        SET_FIELD(it.second, std::vector<std::string>, fillconfmap, set_to_default);
     }
 
     return true;
 }
 
-#define UPDATE_FIELD(FIELD, VALUE, TYPE)                                             \
+#define UPDATE_FIELD(FIELD, VALUE, TYPE, PERSIST)                                    \
     if (strcmp((FIELD).type, #TYPE) == 0) {                                          \
         if (!update((VALUE), *reinterpret_cast<TYPE*>((FIELD).storage))) {           \
             return Status::InvalidArgument(                                          \
@@ -304,10 +349,37 @@ bool init(const char* filename, bool fillconfmap) {
             oss << (*reinterpret_cast<TYPE*>((FIELD).storage));                      \
             (*full_conf_map)[(FIELD).name] = oss.str();                              \
         }                                                                            \
+        if (PERSIST) {                                                               \
+            persist_config(std::string(FIELD.name), VALUE);                          \
+        }                                                                            \
         return Status::OK();                                                         \
     }
 
-Status set_config(const std::string& field, const std::string& value) {
+
+// write config to be_custom.conf
+// the caller need to make sure that the given config is valid
+bool persist_config(const std::string& field, const std::string& value) {
+    // lock to make sure only one thread can modify the be_custom.conf
+    std::lock_guard<std::mutex> l(custom_conf_lock); 
+
+    static const string conffile = string(getenv("DORIS_HOME")) + "/conf/be_custom.conf";
+    Status st = FileSystemUtil::create_file(conffile);
+    if (!st.ok()) {
+        LOG(WARNING) << "failed to create or open be_custom.conf. " << st.get_error_msg();
+        return false;
+    }
+
+    Properties tmp_props;
+    if (!tmp_props.load(conffile.c_str())) {
+        LOG(WARNING) << "failed to load " << conffile;
+        return false;
+    }
+
+    tmp_props.set(field, value);
+    return tmp_props.dump(conffile);
+}
+
+Status set_config(const std::string& field, const std::string& value, bool need_persist) {
     auto it = Register::_s_field_map->find(field);
     if (it == Register::_s_field_map->end()) {
         return Status::NotFound(strings::Substitute("'$0' is not found", field));
@@ -317,11 +389,11 @@ Status set_config(const std::string& field, const std::string& value) {
         return Status::NotSupported(strings::Substitute("'$0' is not support to modify", field));
     }
 
-    UPDATE_FIELD(it->second, value, bool);
-    UPDATE_FIELD(it->second, value, int16_t);
-    UPDATE_FIELD(it->second, value, int32_t);
-    UPDATE_FIELD(it->second, value, int64_t);
-    UPDATE_FIELD(it->second, value, double);
+    UPDATE_FIELD(it->second, value, bool, need_persist);
+    UPDATE_FIELD(it->second, value, int16_t, need_persist);
+    UPDATE_FIELD(it->second, value, int32_t, need_persist);
+    UPDATE_FIELD(it->second, value, int64_t, need_persist);
+    UPDATE_FIELD(it->second, value, double, need_persist);
 
     // The other types are not thread safe to change dynamically.
     return Status::NotSupported(strings::Substitute(
diff --git a/be/src/common/configbase.h b/be/src/common/configbase.h
index 9dd58bb7b62071..93c06ef31f06de 100644
--- a/be/src/common/configbase.h
+++ b/be/src/common/configbase.h
@@ -21,6 +21,7 @@
 #include <stdint.h>
 
 #include <map>
+#include <mutex>
 #include <string>
 #include <vector>
 
@@ -109,22 +110,38 @@ class Register {
 // configuration properties load from config file.
 class Properties {
 public:
-    bool load(const char* filename);
+    // load conf from file, if must_exist is true and file does not exist, return false
+    bool load(const char* filename, bool must_exist = true);
     template <typename T>
-    bool get(const char* key, const char* defstr, T& retval) const;
+
+    // Find the config value by key from `file_conf_map`.
+    // If found, set `retval` to the config value,
+    // or set `retval` to `defstr`
+    bool get_or_default(const char* key, const char* defstr, T& retval) const;
+
+    void set(const std::string& key, const std::string& val);
+
+    // dump props to conf file
+    bool dump(const std::string& conffile);
 
 private:
     std::map<std::string, std::string> file_conf_map;
 };
 
-extern Properties props;
-
 // full configurations.
 extern std::map<std::string, std::string>* full_conf_map;
 
-bool init(const char* filename, bool fillconfmap = false);
+extern std::mutex custom_conf_lock;
+
+// Init the config from `conf_file`.
+// If fillconfmap is true, the updated config will also update the `full_conf_map`.
+// If must_exist is true and `conf_file` does not exist, this function will return false.
+// If set_to_default is true, the config value will be set to default value if not found in `conf_file`.
+bool init(const char* conf_file, bool fillconfmap = false, bool must_exist = true, bool set_to_default = true);
+
+Status set_config(const std::string& field, const std::string& value, bool need_persist = false);
 
-Status set_config(const std::string& field, const std::string& value);
+bool persist_config(const std::string& field, const std::string& value);
 
 } // namespace config
 } // namespace doris
diff --git a/be/src/http/action/update_config_action.cpp b/be/src/http/action/update_config_action.cpp
index 91eb6b0df00b2e..f7ec29583c1cfe 100644
--- a/be/src/http/action/update_config_action.cpp
+++ b/be/src/http/action/update_config_action.cpp
@@ -38,25 +38,53 @@ namespace doris {
 
 const static std::string HEADER_JSON = "application/json";
 
+const static std::string PERSIST_PARAM = "persist";
+
 void UpdateConfigAction::handle(HttpRequest* req) {
     LOG(INFO) << req->debug_string();
 
     Status s;
     std::string msg;
-    if (req->params()->size() != 1) {
+    // We only support set one config at a time, and along with a optional param "persist".
+    // So the number of query params should at most be 2.
+    if (req->params()->size() > 2 || req->params()->size() < 1) {
         s = Status::InvalidArgument("");
-        msg = "Now only support to set a single config once, via 'config_name=new_value'";
+        msg = "Now only support to set a single config once, via 'config_name=new_value', and with an optional parameter 'persist'.";
     } else {
-        DCHECK(req->params()->size() == 1);
-        const std::string& config = req->params()->begin()->first;
-        const std::string& new_value = req->params()->begin()->second;
-        s = config::set_config(config, new_value);
-        if (s.ok()) {
-            LOG(INFO) << "set_config " << config << "=" << new_value << " success";
-        } else {
-            LOG(WARNING) << "set_config " << config << "=" << new_value << " failed";
-            msg = strings::Substitute("set $0=$1 failed, reason: $2", config, new_value,
-                                      s.to_string());
+        if (req->params()->size() == 1) {
+            const std::string& config = req->params()->begin()->first;
+            const std::string& new_value = req->params()->begin()->second;
+            s = config::set_config(config, new_value, false);
+            if (s.ok()) {
+                LOG(INFO) << "set_config " << config << "=" << new_value << " success";
+            } else {
+                LOG(WARNING) << "set_config " << config << "=" << new_value << " failed";
+                msg = strings::Substitute("set $0=$1 failed, reason: $2", config, new_value,
+                        s.to_string());
+            }
+        } else if (req->params()->size() == 2) {
+            if (req->params()->find(PERSIST_PARAM) == req->params()->end()) {
+                s = Status::InvalidArgument("");
+                msg = "Now only support to set a single config once, via 'config_name=new_value', and with an optional parameter 'persist'.";
+            } else {
+                bool need_persist = false;
+                if (req->params()->find(PERSIST_PARAM)->second.compare("true") == 0) {
+                    need_persist = true;
+                }
+                for (auto const& iter : *(req->params())) {
+                    if (iter.first.compare(PERSIST_PARAM) == 0) {
+                        continue;
+                    }
+                    s = config::set_config(iter.first, iter.second, need_persist);
+                    if (s.ok()) {
+                        LOG(INFO) << "set_config " << iter.first << "=" << iter.second << " success. persist: " << need_persist;
+                    } else {
+                        LOG(WARNING) << "set_config " << iter.first << "=" << iter.second << " failed";
+                        msg = strings::Substitute("set $0=$1 failed, reason: $2", iter.first, iter.second,
+                                s.to_string());
+                    }
+                }
+            }
         }
     }
 
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index 8181a57b20306e..179c0b6f43638f 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -119,12 +119,22 @@ int main(int argc, char** argv) {
         exit(-1);
     }
 
+    // init config.
+    // the config in be_custom.conf will overwrite the config in be.conf
+    // Must init custom config after init config, separately.
+    // Because the path of custom config file is defined in be.conf
     string conffile = string(getenv("DORIS_HOME")) + "/conf/be.conf";
-    if (!doris::config::init(conffile.c_str(), true)) {
+    if (!doris::config::init(conffile.c_str(), true, true, true)) {
         fprintf(stderr, "error read config file. \n");
         return -1;
     }
 
+    string custom_conffile = doris::config::custom_config_dir + "/be_custom.conf";
+    if (!doris::config::init(custom_conffile.c_str(), true, false, false)) {
+        fprintf(stderr, "error read custom config file. \n");
+        return -1;
+    }
+
 #if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && !defined(THREAD_SANITIZER)
     // Aggressive decommit is required so that unused pages in the TCMalloc page heap are
     // not backed by physical pages and do not contribute towards memory consumption.
diff --git a/docs/en/administrator-guide/config/be_config.md b/docs/en/administrator-guide/config/be_config.md
index 1f348510b481b4..3a8b68b18a1d5f 100644
--- a/docs/en/administrator-guide/config/be_config.md
+++ b/docs/en/administrator-guide/config/be_config.md
@@ -30,8 +30,17 @@ under the License.
 
 This document mainly introduces the relevant configuration items of BE.
 
+The BE configuration file `be.conf` is usually stored in the `conf/` directory of the BE deployment path. In version 0.14, another configuration file `be_custom.conf` will be introduced. The configuration file is used to record the configuration items that are dynamically configured and persisted by the user during operation.
+
+After the BE process is started, it will read the configuration items in `be.conf` first, and then read the configuration items in `be_custom.conf`. The configuration items in `be_custom.conf` will overwrite the same configuration items in `be.conf`.
+
+The location of the `be_custom.conf` file can be configured in `be.conf` through the `custom_config_dir` configuration item.
+
 ## View configuration items
-(TODO)
+
+Users can view the current configuration items by visiting BE's web page:
+
+`http://be_host:be_webserver_port/varz`
 
 ## Set configuration items
 
@@ -39,42 +48,54 @@ There are two ways to configure BE configuration items:
 
 1. Static configuration
 
-         Add and set configuration items in the `conf/be.conf` file. The configuration items in `be.conf` will be read when BE starts. Configuration items not in `be.conf` will use default values.
+    Add and set configuration items in the `conf/be.conf` file. The configuration items in `be.conf` will be read when BE starts. Configuration items not in `be.conf` will use default values.
 
 2. Dynamic configuration
 
-         After BE starts, the configuration items can be dynamically set with the following commands.
+    After BE starts, the configuration items can be dynamically set with the following commands.
 
-         ```curl -X POST http://{be_ip}:{be_http_port}/api/update_config?{key}={value}'```
+    ```
+    curl -X POST http://{be_ip}:{be_http_port}/api/update_config?{key}={value}'
+    ```
 
-         **Configuration items modified in this way will become invalid after the BE process restarts. **
+    In version 0.13 and before, the configuration items modified in this way will become invalid after the BE process restarts. In 0.14 and later versions, the modified configuration can be persisted through the following command. The modified configuration items are stored in the `be_custom.conf` file.
+
+    ```
+    curl -X POST http://{be_ip}:{be_http_port}/api/update_config?{key}={value}&persis=true'
+    ```
 
 ## Examples
 
 1. Modify `max_compaction_concurrency` statically
 
-         By adding in the `be.conf` file:
+     By adding in the `be.conf` file:
 
-         ```max_compaction_concurrency=5```
+     ```max_compaction_concurrency=5```
 
-         Then restart the BE process to take effect the configuration.
+     Then restart the BE process to take effect the configuration.
 
 2. Modify `streaming_load_max_mb` dynamically
 
-         After BE starts, the configuration item `streaming_load_max_mb` is dynamically set by the following command:
+    After BE starts, the configuration item `streaming_load_max_mb` is dynamically set by the following command:
+
+    ```
+    curl -X POST http://{be_ip}:{be_http_port}/api/update_config?streaming_load_max_mb=1024
+    ```
 
-         ```curl -X POST http://{be_ip}:{be_http_port}/api/update_config?streaming_load_max_mb=1024```
+    The return value is as follows, indicating that the setting is successful.
 
-         The return value is as follows, indicating that the setting is successful.
+    ```
+    {
+        "status": "OK",
+        "msg": ""
+    }
+    ```
 
-         ```
-         {
-             "status": "OK",
-             "msg": ""
-         }
-         ```
+    The configuration will become invalid after the BE restarts. If you want to persist the modified results, use the following command:
 
-         **The configuration will be invalid after BE restarted. **
+    ```
+    curl -X POST http://{be_ip}:{be_http_port}/api/update_config?streaming_load_max_mb=1024\&persist=true
+    ```
 
 ## Configurations
 
@@ -233,6 +254,12 @@ Generally, the configuration is within 512m. If the configuration is too large,
 
 Generally, the configuration is within 128m. Over configuration will cause more cumulative compaction write amplification.
 
+### `custom_config_dir`
+
+Configure the location of the `be_custom.conf` file. The default is in the `conf/` directory.
+
+In some deployment environments, the `conf/` directory may be overwritten due to system upgrades. This will cause the user modified configuration items to be overwritten. At this time, we can store `be_custom.conf` in another specified directory to prevent the configuration file from being overwritten.
+
 ### `default_num_rows_per_column_file_block`
 
 ### `default_query_options`
diff --git a/docs/en/administrator-guide/config/fe_config.md b/docs/en/administrator-guide/config/fe_config.md
index 84340a663bb489..876519c2b1cc05 100644
--- a/docs/en/administrator-guide/config/fe_config.md
+++ b/docs/en/administrator-guide/config/fe_config.md
@@ -30,6 +30,12 @@ under the License.
 
 This document mainly introduces the relevant configuration items of FE.
 
+The FE configuration file `fe.conf` is usually stored in the `conf/` directory of the FE deployment path. In version 0.14, another configuration file `fe_custom.conf` will be introduced. The configuration file is used to record the configuration items that are dynamically configured and persisted by the user during operation.
+
+After the FE process is started, it will read the configuration items in `fe.conf` first, and then read the configuration items in `fe_custom.conf`. The configuration items in `fe_custom.conf` will overwrite the same configuration items in `fe.conf`.
+
+The location of the `fe_custom.conf` file can be configured in `fe.conf` through the `custom_config_dir` configuration item.
+
 ## View configuration items
 
 There are two ways to view the configuration items of FE:
@@ -61,7 +67,7 @@ There are two ways to configure FE configuration items:
 
     Add and set configuration items in the `conf/fe.conf` file. The configuration items in `fe.conf` will be read when the FE process starts. Configuration items not in `fe.conf` will use default values.
     
-2. Dynamic configuration
+2. Dynamic configuration via MySQL protocol
 
     After the FE starts, you can set the configuration items dynamically through the following commands. This command requires administrator privilege.
 
@@ -74,6 +80,12 @@ There are two ways to configure FE configuration items:
     **Configuration items modified in this way will become invalid after the FE process restarts.**
 
     For more help on this command, you can view it through the `HELP ADMIN SET CONFIG;` command.
+    
+3. Dynamic configuration via HTTP protocol
+
+    For details, please refer to [Set Config Action](../http-actions/fe/set-config-action.md)
+
+    This method can also persist the modified configuration items. The configuration items will be persisted in the `fe_custom.conf` file and will still take effect after FE is restarted.
 
 ## Examples
 
@@ -99,7 +111,7 @@ There are two ways to configure FE configuration items:
     set forward_to_master = true;
     ADMIN SHOW FRONTEND CONFIG;
     ```
-    
+
     After modification in the above manner, if the Master FE restarts or a Master election is performed, the configuration will be invalid. You can add the configuration item directly in `fe.conf` and restart the FE to make the configuration item permanent.
 
 3. Modify `max_distribution_pruner_recursion_depth`
@@ -210,6 +222,12 @@ But at the same time, it will cause the submission of failed or failed execution
 
 ### `consistency_check_start_time`
 
+### `custom_config_dir`
+
+Configure the location of the `fe_custom.conf` file. The default is in the `conf/` directory.
+
+In some deployment environments, the `conf/` directory may be overwritten due to system upgrades. This will cause the user modified configuration items to be overwritten. At this time, we can store `fe_custom.conf` in another specified directory to prevent the configuration file from being overwritten.
+
 ### `db_used_data_quota_update_interval_secs`
 
 For better data load performance, in the check of whether the amount of data used by the database before data load exceeds the quota, we do not calculate the amount of data already used by the database in real time, but obtain the periodically updated value of the daemon thread.
diff --git a/docs/en/administrator-guide/http-actions/fe/set-config-action.md b/docs/en/administrator-guide/http-actions/fe/set-config-action.md
index 7986f6ee837368..bc1d786a213aa6 100644
--- a/docs/en/administrator-guide/http-actions/fe/set-config-action.md
+++ b/docs/en/administrator-guide/http-actions/fe/set-config-action.md
@@ -43,6 +43,10 @@ None
 * `confkey1=confvalue1`
 
     Specify the configuration name to be set, and its value is the configuration value to be modified.
+    
+* `persist`
+
+     Whether to persist the modified configuration. The default is false, which means it is not persisted. If it is true, the modified configuration item will be written into the `fe_custom.conf` file and will still take effect after FE is restarted.
 
 ## Request body
 
@@ -90,3 +94,23 @@ The `set` field indicates the successfully set configuration. The `err` field in
     	"count": 0
     }
     ```
+
+2. Set `max_bytes_per_broker_scanner` and persist it.
+
+    ```
+    GET /api/_set_config?max_bytes_per_broker_scanner=21474836480&persist=true
+    
+    Response:
+    {
+    	"msg": "success",
+    	"code": 0,
+    	"data": {
+    		"set": {
+    			"max_bytes_per_broker_scanner": "21474836480"
+    		},
+    		"err": {},
+    		"persist": "ok"
+    	},
+    	"count": 0
+    }
+    ```
\ No newline at end of file
diff --git a/docs/zh-CN/administrator-guide/config/be_config.md b/docs/zh-CN/administrator-guide/config/be_config.md
index 22bab78864d83c..25f3f5465f6b3b 100644
--- a/docs/zh-CN/administrator-guide/config/be_config.md
+++ b/docs/zh-CN/administrator-guide/config/be_config.md
@@ -30,8 +30,15 @@ under the License.
 
 该文档主要介绍 BE 的相关配置项。
 
+BE 的配置文件 `be.conf` 通常存放在 BE 部署路径的 `conf/` 目录下。 而在 0.14 版本中会引入另一个配置文件 `be_custom.conf`。该配置文件用于记录用户在运行是动态配置并持久化的配置项。
+
+BE 进程启动后，会先读取 `be.conf` 中的配置项，之后再读取 `be_custom.conf` 中的配置项。`be_custom.conf` 中的配置项会覆盖 `be.conf` 中相同的配置项。
+
 ## 查看配置项
-（TODO）
+
+用户可以通过访问 BE 的 Web 页面查看当前配置项：
+
+`http://be_host:be_webserver_port/varz`
 
 ## 设置配置项
 
@@ -45,9 +52,15 @@ BE 的配置项有两种方式进行配置：
 
 	BE 启动后，可以通过一下命令动态设置配置项。
 
-	```curl -X POST http://{be_ip}:{be_http_port}/api/update_config?{key}={value}'```
+	```
+	curl -X POST http://{be_ip}:{be_http_port}/api/update_config?{key}={value}'
+	```
 
-	**通过该方式修改的配置项将在 BE 进程重启后失效。**
+	在 0.13 版本及之前，通过该方式修改的配置项将在 BE 进程重启后失效。在 0.14 及之后版本中，可以通过以下命令持久化修改后的配置。修改后的配置项存储在 `be_custom.conf` 文件中。
+	
+	```
+	curl -X POST http://{be_ip}:{be_http_port}/api/update_config?{key}={value}&persist=true'
+	```
 
 ## 应用举例
 
@@ -74,7 +87,11 @@ BE 的配置项有两种方式进行配置：
 	}
 	```
 
-	**BE 重启后该配置将失效。**
+	BE 重启后该配置将失效。如果想持久化修改结果，使用如下命令：
+	
+	```
+	curl -X POST http://{be_ip}:{be_http_port}/api/update_config?streaming_load_max_mb=1024\&persist=true
+	```
 
 ## 配置项列表
 
@@ -231,6 +248,12 @@ Metrics: {"filtered_rows":0,"input_row_num":3346807,"input_rowsets_count":42,"in
 
 一般情况下，配置在128m以内，配置过大会导致cumulative compaction写放大较多。
 
+### `custom_config_dir`
+
+配置 `be_custom.conf` 文件的位置。默认为 `conf/` 目录下。
+
+在某些部署环境下，`conf/` 目录可能因为系统的版本升级被覆盖掉。这会导致用户在运行是持久化修改的配置项也被覆盖。这时，我们可以将 `be_custom.conf` 存储在另一个指定的目录中，以防止配置文件被覆盖。
+
 ### `default_num_rows_per_column_file_block`
 
 ### `default_query_options`
diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md b/docs/zh-CN/administrator-guide/config/fe_config.md
index 0ece531b36bab2..d37d348384f4ee 100644
--- a/docs/zh-CN/administrator-guide/config/fe_config.md
+++ b/docs/zh-CN/administrator-guide/config/fe_config.md
@@ -30,6 +30,12 @@ under the License.
 
 该文档主要介绍 FE 的相关配置项。
 
+FE 的配置文件 `fe.conf` 通常存放在 FE 部署路径的 `conf/` 目录下。 而在 0.14 版本中会引入另一个配置文件 `fe_custom.conf`。该配置文件用于记录用户在运行是动态配置并持久化的配置项。
+
+FE 进程启动后，会先读取 `fe.conf` 中的配置项，之后再读取 `fe_custom.conf` 中的配置项。`fe_custom.conf` 中的配置项会覆盖 `fe.conf` 中相同的配置项。
+
+`fe_custom.conf` 文件的位置可以在 `fe.conf` 通过 `custom_config_dir` 配置项配置。 
+
 ## 查看配置项
 
 FE 的配置项有两种方式进行查看：
@@ -61,7 +67,7 @@ FE 的配置项有两种方式进行配置：
 
     在 `conf/fe.conf` 文件中添加和设置配置项。`fe.conf` 中的配置项会在 FE 进程启动时被读取。没有在 `fe.conf` 中的配置项将使用默认值。
     
-2. 动态配置
+2. 通过 MySQL 协议动态配置
 
     FE 启动后，可以通过以下命令动态设置配置项。该命令需要管理员权限。
     
@@ -75,6 +81,12 @@ FE 的配置项有两种方式进行配置：
     
     更多该命令的帮助，可以通过 `HELP ADMIN SET CONFIG;` 命令查看。
     
+3. 通过 HTTP 协议动态配置
+
+    具体请参阅 [Set Config Action](../http-actions/fe/set-config-action.md)
+    
+    该方式也可以持久化修改后的配置项。配置项将持久化在 `fe_custom.conf` 文件中，在 FE 重启后仍会生效。
+    
 ## 应用举例
 
 1. 修改 `async_load_task_pool_size`
@@ -208,6 +220,12 @@ FE 的配置项有两种方式进行配置：
 
 ### `consistency_check_start_time`
 
+### `custom_config_dir`
+
+配置 `fe_custom.conf` 文件的位置。默认为 `conf/` 目录下。
+
+在某些部署环境下，`conf/` 目录可能因为系统的版本升级被覆盖掉。这会导致用户在运行是持久化修改的配置项也被覆盖。这时，我们可以将 `fe_custom.conf` 存储在另一个指定的目录中，以防止配置文件被覆盖。
+
 ### `db_used_data_quota_update_interval_secs`
 
 为了更好的数据导入性能，在数据导入之前的数据库已使用的数据量是否超出配额的检查中，我们并不实时计算数据库已经使用的数据量，而是获取后台线程周期性更新的值。
diff --git a/docs/zh-CN/administrator-guide/http-actions/fe/set-config-action.md b/docs/zh-CN/administrator-guide/http-actions/fe/set-config-action.md
index c8a769f5e2d620..6b00ceb4a18491 100644
--- a/docs/zh-CN/administrator-guide/http-actions/fe/set-config-action.md
+++ b/docs/zh-CN/administrator-guide/http-actions/fe/set-config-action.md
@@ -32,7 +32,7 @@ under the License.
 
 ## Description
 
-用于动态设置 FE 的参数。该命令等通过 `ADMIN SET FRONTEND CONFIG` 命令。但该命令仅会设置对应 FE 节点的配置。并且不会自动转发 `MasterOnly` 配置项给 Master FE 节点。
+用于动态设置 FE 的参数。该命令等同于通过 `ADMIN SET FRONTEND CONFIG` 命令。但该命令仅会设置对应 FE 节点的配置。并且不会自动转发 `MasterOnly` 配置项给 Master FE 节点。
     
 ## Path parameters
 
@@ -43,6 +43,10 @@ under the License.
 * `confkey1=confvalue1`
 
     指定要设置的配置名称，其值为要修改的配置值。
+    
+* `persist`
+
+    是否要将修改的配置持久化。默认为 false，即不持久化。如果为 true，这修改后的配置项会写入 `fe_custom.conf` 文件中，并在 FE 重启后仍会生效。
 
 ## Request body
 
@@ -90,3 +94,22 @@ under the License.
     	"count": 0
     }
     ```
+
+2. 设置 `max_bytes_per_broker_scanner` 并持久化
+    ```
+    GET /api/_set_config?max_bytes_per_broker_scanner=21474836480&persist=true
+    
+    Response:
+    {
+    	"msg": "success",
+    	"code": 0,
+    	"data": {
+    		"set": {
+    			"max_bytes_per_broker_scanner": "21474836480"
+    		},
+    		"err": {},
+    		"persist": "ok"
+    	},
+    	"count": 0
+    }
+    ```
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java
index 7d8c8a40f5db40..094a7396308775 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java
@@ -81,7 +81,11 @@ public static void start(String dorisHomeDir, String pidDir, String[] args) {
             }
 
             // init config
-            new Config().init(dorisHomeDir + "/conf/fe.conf");
+            Config config = new Config();
+            config.init(dorisHomeDir + "/conf/fe.conf");
+            // Must init custom config after init config, separately.
+            // Because the path of custom config file is defined in fe.conf
+            config.initCustom(Config.custom_config_dir + "/fe_custom.conf");
 
             // check it after Config is initialized, otherwise the config 'check_java_version' won't work.
             if (!JdkUtils.checkJavaVersion()) {
@@ -112,7 +116,6 @@ public static void start(String dorisHomeDir, String pidDir, String[] args) {
             QeService qeService = new QeService(Config.query_port, Config.mysql_service_nio_enabled, ExecuteEnv.getInstance().getScheduler());
             FeServer feServer = new FeServer(Config.rpc_port);
 
-
             feServer.start();
 
             if (!Config.enable_http_server_v2) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index 97c13c8b0c6d8b..091b3b86474617 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -21,6 +21,12 @@
 import org.apache.doris.http.HttpServer;
 
 public class Config extends ConfigBase {
+
+    /**
+     * Dir of custom config file
+     */
+    @ConfField
+    public static String custom_config_dir = PaloFe.DORIS_HOME_DIR + "/conf";
     
     /**
      * The max size of one sys log and audit log
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java b/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java
index f2ec56bfc72875..4943748a209b9d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java
@@ -24,7 +24,11 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.reflect.Field;
@@ -45,17 +49,33 @@ public class ConfigBase {
         boolean mutable() default false;
         boolean masterOnly() default false;
         String comment() default "";
-    }   
-    
-    public static Properties props;
+    }
+
+    private static String confFile;
+    private static String customConfFile;
     public static Class<? extends ConfigBase> confClass;
-    
-    public void init(String profile) throws Exception {
-        props = new Properties();
+
+    public void init(String confFile) throws Exception {
         confClass = this.getClass();
-        props.load(new FileReader(profile));
-        replacedByEnv();
-        setFields();
+        this.confFile = confFile;
+        initConf(confFile);
+    }
+
+    public void initCustom(String customConfFile) throws Exception {
+        this.customConfFile = customConfFile;
+        File file = new File(customConfFile);
+        if (file.exists() && file.isFile()) {
+            // customConfFile is introduced in version 0.14, for compatibility, check if it exist
+            // config in customConfFile will overwrite the config in confFile
+            initConf(customConfFile);
+        }
+    }
+
+    private void initConf(String confFile) throws Exception {
+        Properties props = new Properties();
+        props.load(new FileReader(confFile));
+        replacedByEnv(props);
+        setFields(props);
     }
     
     public static HashMap<String, String> dump() throws Exception { 
@@ -94,9 +114,13 @@ public static HashMap<String, String> dump() throws Exception {
         }
         return map;
     }
-    
-    private static void replacedByEnv() throws Exception {
-        Pattern pattern = Pattern.compile("\\$\\{([^\\}]*)\\}");    
+
+    // there is some config in fe.conf like:
+    // config_key={CONFIG_VALUE}
+    // the "CONFIG_VALUE" should be replaced be env variable CONFIG_VALUE
+    private void replacedByEnv(Properties props) throws Exception {
+        // pattern to match string like "{CONFIG_VALUE}"
+        Pattern pattern = Pattern.compile("\\$\\{([^\\}]*)\\}");
         for (String key : props.stringPropertyNames()) {
             String value = props.getProperty(key);
             Matcher m = pattern.matcher(value);
@@ -109,11 +133,11 @@ private static void replacedByEnv() throws Exception {
                     throw new Exception("no such env variable: " + m.group(1));
                 }
             }
-            props.setProperty(key, value);      
+            props.setProperty(key, value);
         }
     }
-    
-    private static void setFields() throws Exception {      
+
+    private static void setFields(Properties props) throws Exception {
         Field[] fields = confClass.getFields();     
         for (Field f : fields) {
             // ensure that field has "@ConfField" annotation
@@ -284,4 +308,32 @@ public synchronized static boolean checkIsMasterOnly(String key) {
 
         return anno.masterOnly();
     }
+
+    // overwrite configs to customConfFile.
+    // use synchronized to make sure only one thread modify this file
+    public synchronized static void persistConfig(Map<String, String> customConf) throws IOException {
+        File file = new File(customConfFile);
+        if (!file.exists()) {
+            file.createNewFile();
+        } else {
+            // clear the file content
+            try (PrintWriter writer = new PrintWriter(file)) {
+                writer.print("");
+            }
+        }
+
+        Properties props = new Properties();
+        props.load(new FileReader(customConfFile));
+
+        for (Map.Entry<String, String> entry : customConf.entrySet()) {
+            props.setProperty(entry.getKey(), entry.getValue());
+        }
+
+        try (FileOutputStream fos = new FileOutputStream(file)) {
+            props.store(fos, "THIS IS AN AUTO GENERATED CONFIG FILE.\n"
+                    + "You can modify this file manually, and the configurations in this file\n"
+                    + "will overwrite the configurations in fe.conf");
+        }
+    }
 }
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
index ed90c5637f0a23..dc1c74d939b16d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
@@ -17,6 +17,8 @@
 
 package org.apache.doris.http.rest;
 
+import io.netty.handler.codec.http.HttpMethod;
+
 import org.apache.doris.catalog.Catalog;
 import org.apache.doris.common.ConfigBase;
 import org.apache.doris.common.ConfigBase.ConfField;
@@ -34,12 +36,11 @@
 import org.apache.logging.log4j.Logger;
 import org.codehaus.jackson.map.ObjectMapper;
 
+import java.io.IOException;
 import java.lang.reflect.Field;
 import java.util.List;
 import java.util.Map;
 
-import io.netty.handler.codec.http.HttpMethod;
-
 /*
  * used to set fe config
  * eg:
@@ -48,6 +49,8 @@
 public class SetConfigAction extends RestBaseAction {
     private static final Logger LOG = LogManager.getLogger(SetConfigAction.class);
 
+    private static final String PERSIST_PARAM = "persist";
+
     public SetConfigAction(ActionController controller) {
         super(controller);
     }
@@ -61,11 +64,19 @@ public static void registerAction(ActionController controller) throws IllegalArg
     protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
         checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
 
+        boolean needPersist = false;
         Map<String, List<String>> configs = request.getAllParameters();
+        if (configs.containsKey(PERSIST_PARAM)) {
+            List<String> val = configs.remove(PERSIST_PARAM);
+            if (val.size() == 1 && val.get(0).equals("true")) {
+                needPersist = true;
+            }
+        }
+
         Map<String, String> setConfigs = Maps.newHashMap();
         Map<String, String> errConfigs = Maps.newHashMap();
 
-        LOG.debug("get config from url: {}", configs);
+        LOG.debug("get config from url: {}, need persist: {}", configs, needPersist);
 
         Field[] fields = ConfigBase.confClass.getFields();
         for (Field f : fields) {
@@ -100,15 +111,27 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response
             setConfigs.put(confKey, confVals.get(0));
         }
 
+        String persistMsg = "";
+        if (needPersist) {
+            try {
+                ConfigBase.persistConfig(setConfigs);
+                persistMsg = "ok";
+            } catch (IOException e) {
+                LOG.warn("failed to persist config", e);
+                persistMsg = e.getMessage();
+            }
+        }
+
         for (String key : configs.keySet()) {
             if (!setConfigs.containsKey(key)) {
                 errConfigs.put(key, configs.get(key).toString());
             }
         }
 
-        Map<String, Map<String, String>> resultMap = Maps.newHashMap();
+        Map<String, Object> resultMap = Maps.newHashMap();
         resultMap.put("set", setConfigs);
         resultMap.put("err", errConfigs);
+        resultMap.put("persist", persistMsg);
 
         // to json response
         String result = "";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java
index ae547fe224f1e5..7dc591483feb4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java
@@ -24,18 +24,20 @@
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.qe.ConnectContext;
 
+import com.google.common.collect.Maps;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Map;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import java.lang.reflect.Field;
-import java.util.Map;
 
 /*
  * used to set fe config
@@ -46,16 +48,26 @@
 public class SetConfigAction extends RestBaseController {
     private static final Logger LOG = LogManager.getLogger(SetConfigAction.class);
 
+    private static final String PERSIST_PARAM = "persist";
+
     @RequestMapping(path = "/api/_set_config", method = RequestMethod.GET)
     protected Object set_config(HttpServletRequest request, HttpServletResponse response) {
         executeCheckPassword(request, response);
         checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
 
+        boolean needPersist = false;
         Map<String, String[]> configs = request.getParameterMap();
+        if (configs.containsKey(PERSIST_PARAM)) {
+            String[] val = configs.remove(PERSIST_PARAM);
+            if (val.length == 1 && val[0].equals("true")) {
+                needPersist = true;
+            }
+        }
+
         Map<String, String> setConfigs = Maps.newHashMap();
         Map<String, String> errConfigs = Maps.newHashMap();
 
-        LOG.debug("get config from url: {}", configs);
+        LOG.debug("get config from url: {}, need persist: {}", configs, needPersist);
 
         Field[] fields = ConfigBase.confClass.getFields();
         for (Field f : fields) {
@@ -90,6 +102,17 @@ protected Object set_config(HttpServletRequest request, HttpServletResponse resp
             setConfigs.put(confKey, confVals[0]);
         }
 
+        String persistMsg = "";
+        if (needPersist) {
+            try {
+                ConfigBase.persistConfig(setConfigs);
+                persistMsg = "ok";
+            } catch (IOException e) {
+                LOG.warn("failed to persist config", e);
+                persistMsg = e.getMessage();
+            }
+        }
+
         for (String key : configs.keySet()) {
             if (!setConfigs.containsKey(key)) {
                 String[] confVals = configs.get(key);
@@ -98,9 +121,10 @@ protected Object set_config(HttpServletRequest request, HttpServletResponse resp
             }
         }
 
-        Map<String, Map<String, String>> resultMap = Maps.newHashMap();
+        Map<String, Object> resultMap = Maps.newHashMap();
         resultMap.put("set", setConfigs);
         resultMap.put("err", errConfigs);
+        resultMap.put("persist", persistMsg);
 
         return ResponseEntityBuilder.ok(resultMap);
     }