From d9971b5db02e1b970449d8c9d5ae1ac421ce76c7 Mon Sep 17 00:00:00 2001 From: Paul Gaiduk Date: Thu, 31 Oct 2024 21:00:42 +0100 Subject: [PATCH] new metrics for newlogd Add new metrics for newlogd: - LatestAvailableLog - the timestamp of the latest log available on the device - TotalSizeLogs - the total size of all logs on the device Signed-off-by: Paul Gaiduk --- pkg/newlog/cmd/newlogd.go | 57 ++++++++++++++++++++++++--------- pkg/pillar/types/newlogtypes.go | 12 ++++--- 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/pkg/newlog/cmd/newlogd.go b/pkg/newlog/cmd/newlogd.go index ad30fda4a4..c4402ba6d7 100644 --- a/pkg/newlog/cmd/newlogd.go +++ b/pkg/newlog/cmd/newlogd.go @@ -771,6 +771,8 @@ func writelogFile(logChan <-chan inputEntry, moveChan chan fileChanInfo) { defer devStatsKeep.file.Close() devStatsKeep.notUpload = true + logmetrics.DevMetrics.LatestAvailableLog = time.Now() + devSourceBytes = base.NewLockedStringMap() appStatsMap = make(map[string]statsLogFile) checklogTimer := time.NewTimer(5 * time.Second) @@ -1000,41 +1002,48 @@ func checkKeepQuota() { maxSize := int64(limitGzipFilesMbyts * 1000000) sfiles := make(map[string]gfileStats) - key0, size0, err := checkDirGZFiles(sfiles, keepSentDir) + filesKeepSent, sizeKeepSent, err := checkDirGZFiles(sfiles, keepSentDir) if err != nil { log.Errorf("checkKeepQuota: keepSentDir %v", err) } - key1, size1, err := checkDirGZFiles(sfiles, uploadAppDir) + filesAppUpload, sizeAppUpload, err := checkDirGZFiles(sfiles, uploadAppDir) if err != nil { log.Errorf("checkKeepQuota: AppDir %v", err) } - key2, size2, err := checkDirGZFiles(sfiles, uploadDevDir) + filesDevUpload, sizeDevUpload, err := checkDirGZFiles(sfiles, uploadDevDir) if err != nil { log.Errorf("checkKeepQuota: DevDir %v", err) } - key3, size3, err := checkDirGZFiles(sfiles, failSendDir) + fileFailSend, sizeFailSend, err := checkDirGZFiles(sfiles, failSendDir) if err != nil && !os.IsNotExist(err) { log.Errorf("checkKeepQuota: FailToSendDir %v", err) } - totalsize := size0 + size1 + size2 + size3 - totalCount := len(key0) + len(key1) + len(key2) + len(key3) + totalsize := sizeKeepSent + sizeAppUpload + sizeDevUpload + sizeFailSend + totalCount := len(filesKeepSent) + len(filesAppUpload) + len(filesDevUpload) + len(fileFailSend) removed := 0 // limit file count to not as they can have less size than expected // we can have enormous number of files maxCount := int(maxSize / maxGzipFileSize) if totalsize > maxSize || totalCount > maxCount { - keys := key0 - keys = append(keys, key1...) - keys = append(keys, key2...) - keys = append(keys, key3...) - sort.Strings(keys) - - for _, key := range keys { - if _, ok := sfiles[key]; !ok { + allFiles := filesKeepSent + allFiles = append(allFiles, filesAppUpload...) + allFiles = append(allFiles, filesDevUpload...) + allFiles = append(allFiles, fileFailSend...) + + // please notice that the files will be removed in alphabetical order, + // starting from the oldest (lowest) timestamp + // app.log...gz + // dev.log.keep...gz + // dev.log.upload....gz + // TODO: declare a clear order of removal by directory + sort.Strings(allFiles) + + for _, filename := range allFiles { + if _, ok := sfiles[filename]; !ok { continue } - fs := sfiles[key] + fs := sfiles[filename] filePath := filepath.Join(fs.logDir, fs.filename) if _, err := os.Stat(filePath); err != nil { continue @@ -1043,6 +1052,12 @@ func checkKeepQuota() { log.Errorf("checkKeepQuota: remove failed %s, %v", filePath, err) continue } + if fs.logDir == keepSentDir { + // since the files are sorted by name and we delete the oldest files first, + // we can assume that the latest available log (from the file that is next in line to be deleted) + // has the timestamp of the file that was just deleted + logmetrics.DevMetrics.LatestAvailableLog = getTimestampFromGzip(fs.filename) + } if !fs.isSent { logmetrics.NumGZipFileRemoved++ } @@ -1055,6 +1070,18 @@ func checkKeepQuota() { } log.Tracef("checkKeepQuota: %d gzip files removed", removed) } + logmetrics.TotalSizeLogs = uint64(totalsize) +} + +func getTimestampFromGzip(fName string) time.Time { + fName = strings.TrimPrefix(fName, types.DevPrefixKeep) + fName = strings.TrimPrefix(fName, types.DevPrefix) + fName = strings.TrimSuffix(fName, ".gz") + fTime, err := strconv.Atoi(fName) + if err != nil { + log.Fatal(err) + } + return time.Unix(0, int64(fTime)*int64(time.Millisecond)) } func doMoveCompressFile(ps *pubsub.PubSub, tmplogfileInfo fileChanInfo) { diff --git a/pkg/pillar/types/newlogtypes.go b/pkg/pillar/types/newlogtypes.go index 43b74c667f..3be6cbb055 100644 --- a/pkg/pillar/types/newlogtypes.go +++ b/pkg/pillar/types/newlogtypes.go @@ -22,11 +22,12 @@ const ( type logfileMetrics struct { // from newlogd - NumGZipFilesSent uint64 // total gzip files uploaded - NumGZipBytesWrite uint64 // total gzip log in bytes - NumBytesWrite uint64 // total log bytes write to file before gzip - NumGzipFileInDir uint32 // current number of gzip files remain - NumInputEvent uint64 // total event input from log source + NumGZipFilesSent uint64 // total gzip files uploaded + NumGZipBytesWrite uint64 // total gzip log in bytes + NumBytesWrite uint64 // total log bytes write to file before gzip + NumGzipFileInDir uint32 // current number of gzip files remain + NumInputEvent uint64 // total event input from log source + LatestAvailableLog time.Time // latest log timestamp available on device // from loguploader NumGZipFileRetry uint64 // total gzip file upload retries NumGZipFileKeptLocal uint32 // total gzip file upload 4xx failure and kept on device @@ -69,6 +70,7 @@ type NewlogMetrics struct { NumKmessages uint64 // total input kmessages NumSyslogMessages uint64 // total input syslog message DevTop10InputBytesPCT map[string]uint32 // top 10 sources device log input in percentage + TotalSizeLogs uint64 // total size of logs on device // upload latency Latency cloudDelay