fix time parsing problem (#552)

* fix time parsing problem
alibaba · Jan 10, 2023 · c90d573 · c90d573
1 parent 59667ee
commit c90d573
Show file tree

Hide file tree

Showing 8 changed files with 422 additions and 47 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -35,3 +35,5 @@ your changes, such as:
 - [public] [both] [updated] add a new feature
 
 ## [Unreleased]
+
+- [public] [both] [fixed] ignore time zone adjustment in config when using system time as log time
diff --git a/core/config_manager/ConfigManagerBase.cpp b/core/config_manager/ConfigManagerBase.cpp
@@ -669,6 +669,8 @@ void ConfigManagerBase::LoadSingleUserConfig(const std::string& logName, const J
                     config->mUnAcceptDirPattern = GetStringVector(value["dir_pattern_black_list"]);
                 }
 
+                // TODO: the following codes (line 673 - line 724) seem to be the same as the codes in line 816 - line
+                // 874. Remove the following codes in the future.
                 if (value.isMember("merge_type") && value["merge_type"].isString()) {
                     string mergeType = value["merge_type"].asString();
 
@@ -828,17 +830,35 @@ void ConfigManagerBase::LoadSingleUserConfig(const std::string& logName, const J
                 string logTZ = value["log_tz"].asString();
                 int logTZSecond = 0;
                 bool adjustFlag = value["tz_adjust"].asBool();
-                if (adjustFlag && !ParseTimeZoneOffsetSecond(logTZ, logTZSecond)) {
-                    LOG_ERROR(sLogger, ("invalid log time zone set", logTZ));
-                    config->mTimeZoneAdjust = false;
+                if (adjustFlag) {
+                    if (config->mTimeFormat.empty()) {
+                        LOG_WARNING(sLogger,
+                                    ("choose to adjust log time zone, but time format is not specified",
+                                     "use system time as log time instead")("project", config->mProjectName)(
+                                        "logstore", config->mCategory)("config", config->mConfigName));
+                        config->mTimeZoneAdjust = false;
+                    } else if ((logType == DELIMITER_LOG || logType == JSON_LOG) && config->mTimeKey.empty()) {
+                        LOG_WARNING(sLogger,
+                                    ("choose to adjust log time zone, but time key is not specified",
+                                     "use system time as log time instead")("project", config->mProjectName)(
+                                        "logstore", config->mCategory)("config", config->mConfigName));
+                        config->mTimeZoneAdjust = false;
+                    } else if (!ParseTimeZoneOffsetSecond(logTZ, logTZSecond)) {
+                        LOG_WARNING(sLogger,
+                                    ("invalid log time zone specified, will parse log time without time zone adjusted",
+                                     logTZ)("project", config->mProjectName)("logstore", config->mCategory)(
+                                        "config", config->mConfigName));
+                        config->mTimeZoneAdjust = false;
+                    } else {
+                        config->mTimeZoneAdjust = adjustFlag;
+                        config->mLogTimeZoneOffsetSecond = logTZSecond;
+                        LOG_INFO(sLogger,
+                                 ("set log time zone", logTZ)("project", config->mProjectName)(
+                                     "logstore", config->mCategory)("config", config->mConfigName));
+                    }
                 } else {
                     config->mTimeZoneAdjust = adjustFlag;
                     config->mLogTimeZoneOffsetSecond = logTZSecond;
-                    if (adjustFlag) {
-                        LOG_INFO(sLogger,
-                                 ("set log timezone adjust, project", config->mProjectName)(
-                                     "logstore", config->mCategory)("time zone", logTZ)("offset seconds", logTZSecond));
-                    }
                 }
             }
 

diff --git a/core/parser/LogParser.cpp b/core/parser/LogParser.cpp
@@ -379,7 +379,6 @@ bool LogParser::RegexLogLineParser(const char* buffer,
         return true;
     } else if (!discardUnmatch) {
         LogParser::AddUnmatchLog(buffer, logGroup, logGroupSize);
-        return true;
     }
     return false;
 }
@@ -448,12 +447,10 @@ bool LogParser::RegexLogLineParser(const char* buffer,
         parseSuccess = false;
     }
     if (!parseSuccess) {
-        if (discardUnmatch)
-            return false;
-        else {
+        if (!discardUnmatch) {
             AddUnmatchLog(buffer, logGroup, logGroupSize);
-            return true;
         }
+        return false;
     }
 
     Log* logPtr = logGroup.add_logs();
@@ -524,7 +521,7 @@ bool LogParser::ParseLogTime(const char* buffer,
     }
 
     if (logTime <= 0
-        || (BOOL_FLAG(ilogtail_discard_old_data) && (time(NULL) - logTime) > INT32_FLAG(ilogtail_discard_interval))) {
+        || (BOOL_FLAG(ilogtail_discard_old_data) && (time(NULL) - logTime + tzOffsetSecond) > INT32_FLAG(ilogtail_discard_interval))) {
         if (AppConfig::GetInstance()->IsLogParseAlarmValid()) {
             if (LogtailAlarm::GetInstance()->IsLowLevelAlarmValid()) {
                 LOG_WARNING(sLogger,
@@ -694,14 +691,13 @@ bool LogParser::ApsaraEasyReadLogLineParser(const char* buffer,
             PARSE_TIME_FAIL_ALARM, bufOut + " $ " + ToString(logTime), projectName, category, region);
         error = PARSE_LOG_TIMEFORMAT_ERROR;
 
-        if (discardUnmatch)
-            return false;
-        else {
+        if (!discardUnmatch)
+        {
             AddUnmatchLog(buffer, logGroup, logGroupSize);
-            return true;
         }
+        return false;
     }
-    if (BOOL_FLAG(ilogtail_discard_old_data) && (time(NULL) - logTime) > INT32_FLAG(ilogtail_discard_interval)) {
+    if (BOOL_FLAG(ilogtail_discard_old_data) && (time(NULL) - logTime + tzOffsetSecond) > INT32_FLAG(ilogtail_discard_interval)) {
         if (AppConfig::GetInstance()->IsLogParseAlarmValid()) {
             string bufOut(buffer);
             if (bufOut.size() > (size_t)(1024)) {

diff --git a/core/processor/LogProcess.cpp b/core/processor/LogProcess.cpp
@@ -413,8 +413,9 @@ void* LogProcess::ProcessLoop(int32_t threadNo) {
                 int32_t successLogSize = 0;
                 int32_t parseStartTime = (int32_t)time(NULL);
                 for (uint32_t i = 0; i < lines; i++) {
-                    if (!logFileReader->ParseLogLine(
-                            buffer + logIndex[i], logGroup, error, lastLogLineTime, lastLogTimeStr, logGroupSize)) {
+                    bool successful = logFileReader->ParseLogLine(
+                            buffer + logIndex[i], logGroup, error, lastLogLineTime, lastLogTimeStr, logGroupSize);
+                    if (!successful) {
                         ++parseFailures;
                         if (error == PARSE_LOG_REGEX_ERROR)
                             ++regexMatchFailures;
@@ -433,7 +434,7 @@ void* LogProcess::ProcessLoop(int32_t threadNo) {
                                 LogParser::AddLog(
                                     logPtr, config->mAdvancedConfig.mRawLogTag, buffer + logIndex[i], logGroupSize);
                             }
-                            if (config->mTimeZoneAdjust) {
+                            if (successful && config->mTimeZoneAdjust) {
                                 LogParser::AdjustLogTime(
                                     logPtr, config->mLogTimeZoneOffsetSecond, localTimeZoneOffsetSecond);
                             }

diff --git a/core/reader/DelimiterLogFileReader.cpp b/core/reader/DelimiterLogFileReader.cpp
@@ -240,7 +240,6 @@ bool DelimiterLogFileReader::ParseLogLine(const char* buffer,
         return true;
     } else if (!mDiscardUnmatch) {
         LogParser::AddUnmatchLog(buffer, logGroup, logGroupSize);
-        return true;
     }
     return false;
 }

diff --git a/core/reader/JsonLogFileReader.cpp b/core/reader/JsonLogFileReader.cpp
@@ -155,7 +155,6 @@ bool JsonLogFileReader::ParseLogLine(const char* buffer,
         return true;
     } else if (!mDiscardUnmatch) {
         LogParser::AddUnmatchLog(buffer, logGroup, logGroupSize);
-        return true;
     }
     return false;
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -35,3 +35,5 @@ your changes, such as:
		- [public] [both] [updated] add a new feature

		## [Unreleased]

		- [public] [both] [fixed] ignore time zone adjustment in config when using system time as log time