From 4c644eb05babc58fd44239ed89ee7050c04f8ab7 Mon Sep 17 00:00:00 2001 From: IoCing <935600045@qq.com> Date: Thu, 8 Dec 2022 23:53:08 +0800 Subject: [PATCH 1/7] add config about logcleaner --- kvrocks.conf | 9 +++++++++ src/config/config.cc | 2 ++ src/config/config.h | 2 ++ src/main.cc | 9 +++++++++ 4 files changed, 22 insertions(+) diff --git a/kvrocks.conf b/kvrocks.conf index b58827ea48d..01463cc63e3 100644 --- a/kvrocks.conf +++ b/kvrocks.conf @@ -96,6 +96,15 @@ dir /tmp/kvrocks # We also can send logs to stdout/stderr is as simple as: # log-dir stdout +# log-dir log + +# To enable the log cleaner. if set yes , glog will check if there are +# overdue logs whenever a flush is performed +enablelogcleaner yes + +# INFO level log file whose last modified time is greater than logcleanerday +# will be unlinked +logcleanerday 0 # When running in daemonize mode, kvrocks writes a PID file in ${CONFIG_DIR}/kvrocks.pid by # default. You can specify a custom pid file location here. diff --git a/src/config/config.cc b/src/config/config.cc index 8982a7a98c5..30758b44106 100644 --- a/src/config/config.cc +++ b/src/config/config.cc @@ -159,6 +159,8 @@ Config::Config() { {"migrate-sequence-gap", false, new IntField(&sequence_gap, 10000, 1, INT_MAX)}, {"unixsocket", true, new StringField(&unixsocket, "")}, {"unixsocketperm", true, new OctalField(&unixsocketperm, 0777, 1, INT_MAX)}, + {"enablelogcleaner", false, new YesNoField(&enablelogcleaner, false)}, + {"logcleanerday", false, new IntField(&logcleanerday, 365, 0, INT_MAX)}, /* rocksdb options */ {"rocksdb.compression", false, new EnumField(&RocksDB.compression, compression_type_enum, 0)}, diff --git a/src/config/config.h b/src/config/config.h index e57145260b0..7879bca23d2 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -139,6 +139,8 @@ struct Config { int pipeline_size; int sequence_gap; + bool enablelogcleaner; + int logcleanerday; // profiling int profiling_sample_ratio = 0; int profiling_sample_record_threshold_ms = 0; diff --git a/src/main.cc b/src/main.cc index e8d1bb44319..2385e498b80 100644 --- a/src/main.cc +++ b/src/main.cc @@ -178,6 +178,15 @@ static void initGoogleLog(const Config *config) { FLAGS_logtostdout = true; } else { FLAGS_log_dir = config->log_dir; + if (config->enablelogcleaner) { + google::EnableLogCleaner(config->logcleanerday); + google::SetLogFilenameExtension(".log"); + for (int level = google::INFO; level <= google::FATAL; level++) { + std::string des = google::LogSeverityNames[level]; + des = config->log_dir + "/kvrocks_" + des + "_"; + google::SetLogDestination(level, des.c_str()); + } + } } } From 2ddb300b316486af20cb202c00d9c6e8fed27408 Mon Sep 17 00:00:00 2001 From: IoCing <935600045@qq.com> Date: Fri, 9 Dec 2022 11:03:56 +0800 Subject: [PATCH 2/7] revise --- kvrocks.conf | 14 +++++++------- src/config/config.cc | 3 +-- src/config/config.h | 1 - src/main.cc | 10 ++-------- 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/kvrocks.conf b/kvrocks.conf index 01463cc63e3..b7741dd95b9 100644 --- a/kvrocks.conf +++ b/kvrocks.conf @@ -98,13 +98,13 @@ dir /tmp/kvrocks log-dir stdout # log-dir log -# To enable the log cleaner. if set yes , glog will check if there are -# overdue logs whenever a flush is performed -enablelogcleaner yes - -# INFO level log file whose last modified time is greater than logcleanerday -# will be unlinked -logcleanerday 0 +# You can configure logcleanerday to control whether to enable logcleaner +# and the maximum number of days that the INFO level logs will be kept. +# if set logcleanerday to -1 , this mean disable logcleaner. +# if set logcleanerday between 0 to INT_MAX , this mean enable logcleaner , +# and INFO level log file whose last modified time is greater than logcleanerday will be unlinked. +# By default the logcleanerday is -1. +logcleanerday -1 # When running in daemonize mode, kvrocks writes a PID file in ${CONFIG_DIR}/kvrocks.pid by # default. You can specify a custom pid file location here. diff --git a/src/config/config.cc b/src/config/config.cc index 30758b44106..935cd83994f 100644 --- a/src/config/config.cc +++ b/src/config/config.cc @@ -159,8 +159,7 @@ Config::Config() { {"migrate-sequence-gap", false, new IntField(&sequence_gap, 10000, 1, INT_MAX)}, {"unixsocket", true, new StringField(&unixsocket, "")}, {"unixsocketperm", true, new OctalField(&unixsocketperm, 0777, 1, INT_MAX)}, - {"enablelogcleaner", false, new YesNoField(&enablelogcleaner, false)}, - {"logcleanerday", false, new IntField(&logcleanerday, 365, 0, INT_MAX)}, + {"logcleanerday", false, new IntField(&logcleanerday, -1, -1, INT_MAX)}, /* rocksdb options */ {"rocksdb.compression", false, new EnumField(&RocksDB.compression, compression_type_enum, 0)}, diff --git a/src/config/config.h b/src/config/config.h index 7879bca23d2..4fa21e15b32 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -139,7 +139,6 @@ struct Config { int pipeline_size; int sequence_gap; - bool enablelogcleaner; int logcleanerday; // profiling int profiling_sample_ratio = 0; diff --git a/src/main.cc b/src/main.cc index 2385e498b80..1f91ae784c8 100644 --- a/src/main.cc +++ b/src/main.cc @@ -177,15 +177,9 @@ static void initGoogleLog(const Config *config) { FLAGS_stderrthreshold = google::ERROR; FLAGS_logtostdout = true; } else { - FLAGS_log_dir = config->log_dir; - if (config->enablelogcleaner) { + FLAGS_log_dir = config->log_dir + "/"; + if (config->logcleanerday != -1) { google::EnableLogCleaner(config->logcleanerday); - google::SetLogFilenameExtension(".log"); - for (int level = google::INFO; level <= google::FATAL; level++) { - std::string des = google::LogSeverityNames[level]; - des = config->log_dir + "/kvrocks_" + des + "_"; - google::SetLogDestination(level, des.c_str()); - } } } } From b3f2afb4e5b698c265244cbf84a43f47f2406697 Mon Sep 17 00:00:00 2001 From: IoCing <935600045@qq.com> Date: Fri, 9 Dec 2022 12:15:52 +0800 Subject: [PATCH 3/7] revise var name && add callback --- kvrocks.conf | 13 +++++++------ src/config/config.cc | 11 ++++++++++- src/config/config.h | 2 +- src/main.cc | 4 ++-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/kvrocks.conf b/kvrocks.conf index b7741dd95b9..302be382212 100644 --- a/kvrocks.conf +++ b/kvrocks.conf @@ -98,13 +98,14 @@ dir /tmp/kvrocks log-dir stdout # log-dir log -# You can configure logcleanerday to control whether to enable logcleaner +# You can configure log-retention-days to control whether to enable logcleaner # and the maximum number of days that the INFO level logs will be kept. -# if set logcleanerday to -1 , this mean disable logcleaner. -# if set logcleanerday between 0 to INT_MAX , this mean enable logcleaner , -# and INFO level log file whose last modified time is greater than logcleanerday will be unlinked. -# By default the logcleanerday is -1. -logcleanerday -1 +# if set to -1 , this mean disable logcleaner. +# if set to between 0 to INT_MAX , this mean enable logcleaner , +# and INFO level log file whose last modified time is greater than log-retention-days will be unlinked. +# if set to 0 , during the flush, the previous INFO level logs will be immediately unlinked. +# By default the log-retention-days is -1. +log-retention-days -1 # When running in daemonize mode, kvrocks writes a PID file in ${CONFIG_DIR}/kvrocks.pid by # default. You can specify a custom pid file location here. diff --git a/src/config/config.cc b/src/config/config.cc index 935cd83994f..0cb3285abd4 100644 --- a/src/config/config.cc +++ b/src/config/config.cc @@ -159,7 +159,7 @@ Config::Config() { {"migrate-sequence-gap", false, new IntField(&sequence_gap, 10000, 1, INT_MAX)}, {"unixsocket", true, new StringField(&unixsocket, "")}, {"unixsocketperm", true, new OctalField(&unixsocketperm, 0777, 1, INT_MAX)}, - {"logcleanerday", false, new IntField(&logcleanerday, -1, -1, INT_MAX)}, + {"log-retention-days", false, new IntField(&log_retention_days, -1, -1, INT_MAX)}, /* rocksdb options */ {"rocksdb.compression", false, new EnumField(&RocksDB.compression, compression_type_enum, 0)}, @@ -458,6 +458,15 @@ void Config::initFieldCallback() { if (cluster_enabled) srv->slot_migrate_->SetSequenceGapSize(sequence_gap); return Status::OK(); }}, + {"log-retention-days", + [this](Server *srv, const std::string &k, const std::string &v) -> Status{ + if(log_retention_days!=-1){ + google::EnableLogCleaner(log_retention_days); + }else{ + google::DisableLogCleaner(); + } + return Status::OK(); + }}, {"rocksdb.target_file_size_base", [this](Server *srv, const std::string &k, const std::string &v) -> Status { if (!srv) return Status::OK(); diff --git a/src/config/config.h b/src/config/config.h index 4fa21e15b32..04d0cc1056e 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -139,7 +139,7 @@ struct Config { int pipeline_size; int sequence_gap; - int logcleanerday; + int log_retention_days; // profiling int profiling_sample_ratio = 0; int profiling_sample_record_threshold_ms = 0; diff --git a/src/main.cc b/src/main.cc index 1f91ae784c8..8e23b42e593 100644 --- a/src/main.cc +++ b/src/main.cc @@ -178,8 +178,8 @@ static void initGoogleLog(const Config *config) { FLAGS_logtostdout = true; } else { FLAGS_log_dir = config->log_dir + "/"; - if (config->logcleanerday != -1) { - google::EnableLogCleaner(config->logcleanerday); + if (config->log_retention_days != -1) { + google::EnableLogCleaner(config->log_retention_days); } } } From 9a58d2359165ffe3a7cf58be6e713d7c8ec034d7 Mon Sep 17 00:00:00 2001 From: IoCing <935600045@qq.com> Date: Fri, 9 Dec 2022 12:39:22 +0800 Subject: [PATCH 4/7] format --- src/config/config.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config/config.cc b/src/config/config.cc index 0cb3285abd4..d6a6d7c66ab 100644 --- a/src/config/config.cc +++ b/src/config/config.cc @@ -459,10 +459,10 @@ void Config::initFieldCallback() { return Status::OK(); }}, {"log-retention-days", - [this](Server *srv, const std::string &k, const std::string &v) -> Status{ - if(log_retention_days!=-1){ + [this](Server *srv, const std::string &k, const std::string &v) -> Status { + if (log_retention_days != -1) { google::EnableLogCleaner(log_retention_days); - }else{ + } else { google::DisableLogCleaner(); } return Status::OK(); From 4eda41b7989238e16c0a654de5fb8c259931e269 Mon Sep 17 00:00:00 2001 From: IoCing <39056297+IoCing@users.noreply.github.com> Date: Fri, 9 Dec 2022 14:22:58 +0800 Subject: [PATCH 5/7] Update src/config/config.cc Co-authored-by: hulk --- src/config/config.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/config/config.cc b/src/config/config.cc index d6a6d7c66ab..cf3d0ea4594 100644 --- a/src/config/config.cc +++ b/src/config/config.cc @@ -460,6 +460,11 @@ void Config::initFieldCallback() { }}, {"log-retention-days", [this](Server *srv, const std::string &k, const std::string &v) -> Status { + if (!srv) return Status::OK(); + if (Util::ToLower(log_dir) == "stdout") { + return {Status::NotOK, "can't set the 'log-retention-days' when the log dir is stdout"}; + } + if (log_retention_days != -1) { google::EnableLogCleaner(log_retention_days); } else { From 35ef5e779916aa145a733a1c047a0e08280944a3 Mon Sep 17 00:00:00 2001 From: IoCing <39056297+IoCing@users.noreply.github.com> Date: Fri, 9 Dec 2022 14:23:08 +0800 Subject: [PATCH 6/7] Update kvrocks.conf Co-authored-by: hulk --- kvrocks.conf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kvrocks.conf b/kvrocks.conf index 302be382212..e8bd2a50b29 100644 --- a/kvrocks.conf +++ b/kvrocks.conf @@ -96,14 +96,14 @@ dir /tmp/kvrocks # We also can send logs to stdout/stderr is as simple as: # log-dir stdout -# log-dir log - -# You can configure log-retention-days to control whether to enable logcleaner -# and the maximum number of days that the INFO level logs will be kept. -# if set to -1 , this mean disable logcleaner. -# if set to between 0 to INT_MAX , this mean enable logcleaner , -# and INFO level log file whose last modified time is greater than log-retention-days will be unlinked. -# if set to 0 , during the flush, the previous INFO level logs will be immediately unlinked. + +# You can configure log-retention-days to control whether to enable the log cleaner +# and the maximum retention days that the INFO level logs will be kept. +# +# if set to -1, that means to disable the log cleaner. +# if set to 0, all previous INFO level logs will be immediately removed. +# if set to between 0 to INT_MAX, that means it will retent latest N(log-retention-days) day logs. + # By default the log-retention-days is -1. log-retention-days -1 From bec4e30a7b531ee915c1d0d0b499b37a00383b05 Mon Sep 17 00:00:00 2001 From: IoCing <935600045@qq.com> Date: Wed, 21 Dec 2022 00:57:04 +0800 Subject: [PATCH 7/7] add logclean test --- tests/gocase/unit/log/logclean_test.go | 68 ++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tests/gocase/unit/log/logclean_test.go diff --git a/tests/gocase/unit/log/logclean_test.go b/tests/gocase/unit/log/logclean_test.go new file mode 100644 index 00000000000..993ee42d03e --- /dev/null +++ b/tests/gocase/unit/log/logclean_test.go @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package log + +import ( + "os" + "testing" + "time" + + "github.com/apache/incubator-kvrocks/tests/gocase/util" + "github.com/stretchr/testify/require" +) + +func TestLogClean(t *testing.T) { + err := os.Mkdir("/tmp/kvrocks/logfile", os.ModePerm) + if err != nil { + return + } + srv1 := util.StartServer(t, map[string]string{ + "log-dir": "/tmp/kvrocks/logfile", + "log-retention-days": "0", + }) + srv1.Close() + files1, _ := os.ReadDir("/tmp/kvrocks/logfile") + time.Sleep(2000 * time.Millisecond) + srv2 := util.StartServer(t, map[string]string{ + "log-dir": "/tmp/kvrocks/logfile", + "log-retention-days": "0", + }) + srv2.Close() + files2, _ := os.ReadDir("/tmp/kvrocks/logfile") + islogclear := false + for _, f1 := range files1 { + ishave := false + for _, f2 := range files2 { + if f1.Name() == f2.Name() { + ishave = true + break + } + } + if !ishave { + islogclear = true + break + } + } + require.Equal(t, true, islogclear) + err = os.RemoveAll("/tmp/kvrocks/logfile") + if err != nil { + return + } +}