devUpload should not be parsed by egde-view

Since keepSentQueue contains the same or more logs than
devUpload, it's enough to only use those logs when
searching with edge-view.

Signed-off-by: Paul Gaiduk <paulg@zededa.com>

pkg/edgeview/go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.8.4
 	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e
 	github.com/vishvananda/netlink v1.2.1-beta.2
 	golang.org/x/sys v0.18.0
@@ -20,6 +21,7 @@ require (
 
 require (
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/eriknordmark/ipinfo v0.0.0-20230728132417-2d8f4da903d7 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -32,6 +34,7 @@ require (
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d // indirect
 	github.com/miekg/dns v1.1.43 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
@@ -41,4 +44,5 @@ require (
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

pkg/edgeview/src/edgeview_test.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2024 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	. "github.com/onsi/gomega"
+)
+
+func TestWalkLogDirs(t *testing.T) {
+	g := NewWithT(t)
+
+	// test the walkLogDirs function
+	newlogDir = "../../newlog/testdata"
+
+	const layout = "2006-01-02 15:04:05.000 -0700 MST"
+	timestamp := "2024-11-13 10:58:52.618 +0100 CET"
+	parsedTime, err := time.Parse(layout, timestamp)
+	g.Expect(err).NotTo(HaveOccurred(), "failed to parse timestamp")
+
+	from := parsedTime.Add(-1 * time.Second)
+	to := parsedTime.Add(1 * time.Second)
+	foundFiles := walkLogDirs(to.Unix(), from.Unix())
+	g.Expect(foundFiles).To(HaveLen(1), "expected exactly one file to be found")
+
+	expected := logfiletime{
+		filepath: path.Join(newlogDir, "keepSentQueue/dev.log.keep.1731491932618.gz"),
+		filesec:  1731491932,
+	}
+	g.Expect(foundFiles[0]).To(Equal(expected))
+}

pkg/edgeview/src/log-search.go

@@ -10,15 +10,19 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/lf-edge/eve-api/go/logs"
+	"github.com/lf-edge/eve/pkg/pillar/types"
 )
 
+var newlogDir = "/persist/newlog"
+
 type logfiletime struct {
 	filepath string
 	filesec  int64
@@ -75,7 +79,7 @@ func runLogSearch(cmds cmdOpt) {
 		return
 	}
 
-	gfiles := walkLogDirs(t1, t2, now)
+	gfiles := walkLogDirs(t1, t2)
 	prog1 := "zcat"
 	prog2 := "grep"
 	arg2 := []string{"-E", pattern}
@@ -106,66 +110,52 @@ func runLogSearch(cmds cmdOpt) {
 	fmt.Println()
 }
 
-func walkLogDirs(t1, t2, now int64) []logfiletime {
+func walkLogDirs(toTimestamp, fromTimestamp int64) []logfiletime {
 	var getfiles []logfiletime
 
-	files, err := os.ReadDir("/persist/newlog")
+	subdirs, err := os.ReadDir(newlogDir)
 	if err != nil {
-		fmt.Printf("read /persist/newlog error %v\n", err)
+		fmt.Printf("read %s error %v\n", newlogDir, err)
 		return getfiles
 	}
 
-	gzfiles := make(map[string][]string)
-	for _, dir := range files {
-		if !dir.IsDir() {
-			continue
-		}
-		if strings.Contains(dir.Name(), "collect") || strings.Contains(dir.Name(), "panic") {
-			continue
-		}
-		if strings.Contains(dir.Name(), "devUpload") && querytype == "app" {
-			continue
-		}
-		if strings.Contains(dir.Name(), "appUpload") && querytype == "dev" {
+	excludeDirs := []string{"collect", "panic", "devUpload"}
+	if querytype == "dev" {
+		excludeDirs = append(excludeDirs, "appUpload")
+	}
+	excludeFiles := []string{}
+	if querytype == "app" {
+		excludeFiles = append(excludeFiles, "dev")
+	}
+	if querytype == "dev" {
+		excludeFiles = append(excludeFiles, "app")
+	}
+
+	for _, dir := range subdirs {
+		if filterDir(dir, excludeDirs) {
 			continue
 		}
-		files1, err := os.ReadDir("/persist/newlog/" + dir.Name())
+
+		files, err := os.ReadDir(path.Join(newlogDir, dir.Name()))
 		if err != nil {
+			fmt.Printf("read %s error %v\n", path.Join(newlogDir, dir.Name()), err)
 			continue
 		}
-		var groupfiles []string
-		for _, f := range files1 {
-			info, err := f.Info()
-			if err != nil {
+		for _, f := range files {
+			if filterFile(f, excludeFiles) {
 				continue
 			}
-			if info.ModTime().Unix() > t1 || info.ModTime().Unix() < t2 {
-				continue
-			}
-			groupfiles = append(groupfiles, f.Name())
-		}
-		gzfiles["/persist/newlog/"+dir.Name()] = groupfiles
-	}
 
-	for k, g := range gzfiles {
-		for _, l := range g {
-			if !strings.Contains(l, "dev") && !strings.Contains(l, "app") {
-				continue
-			}
-			if querytype == "app" && !strings.Contains(l, "app") {
-				continue
-			}
-			if querytype == "dev" && !strings.Contains(l, "dev") {
-				continue
-			}
-			ftime := getFileTime(l)
-			if ftime == 0 {
+			timestamp, err := types.GetTimestampFromGzipName(f.Name())
+			if err != nil {
 				continue
 			}
-			if ftime >= t2 && ftime <= t1 {
-				file1 := strings.TrimPrefix(l, "./")
+			ftime := timestamp.Unix() // convert to seconds
+
+			if ftime >= fromTimestamp && ftime <= toTimestamp {
+				file1 := strings.TrimPrefix(f.Name(), "./")
 				gfile := logfiletime{
-					filepath: k + "/" + file1,
+					filepath: path.Join(newlogDir, dir.Name(), file1),
 					filesec:  ftime,
 				}
 				getfiles = append(getfiles, gfile)
@@ -180,6 +170,30 @@ func walkLogDirs(t1, t2, now int64) []logfiletime {
 	return getfiles
 }
 
+func filterDir(dir os.DirEntry, filter []string) bool {
+	if !dir.IsDir() {
+		return true
+	}
+	for _, name := range filter {
+		if strings.Contains(dir.Name(), name) {
+			return true
+		}
+	}
+	return false
+}
+
+func filterFile(file os.DirEntry, filter []string) bool {
+	if file.IsDir() {
+		return true
+	}
+	for _, name := range filter {
+		if strings.Contains(file.Name(), name) {
+			return true
+		}
+	}
+	return false
+}
+
 func searchLiveLogs(pattern string, now int64, typeStr string, idx *int, logjson bool) {
 	files, err := os.ReadDir("/persist/newlog/collect")
 	if err != nil {

pkg/edgeview/src/system.go

@@ -160,10 +160,12 @@ func getLogStats() {
 				isDev = true
 			}
 
-			time1 := getFileTime(l.Name())
-			if time1 == 0 {
+			timestamp, err := types.GetTimestampFromGzipName(l.Name())
+			if err != nil {
 				continue
 			}
+			time1 := timestamp.Unix() // convert to seconds
+
 			if isDev && (tmin == 0 || tmin > time1) {
 				tmin = time1
 			}
@@ -308,22 +310,6 @@ func du(currentPath string, info os.FileInfo) int64 {
 	return size
 }
 
-func getFileTime(filename string) int64 {
-	var fn []string
-	if strings.Contains(filename, ".gz") && strings.Contains(filename, ".log.") {
-		fn = strings.Split(filename, ".gz")
-	}
-	if len(fn) < 2 {
-		return 0
-	}
-	fn = strings.Split(fn[0], ".log.")
-	if len(fn) < 2 {
-		return 0
-	}
-	filetime, _ := strconv.Atoi(fn[1])
-	return int64(filetime / 1000)
-}
-
 func getVolume() {
 	jfiles, err := listJSONFiles("/run/zedagent/AppInstanceConfig")
 	if err != nil {