feat: 获取应用日志

cnb-builder-shim/internal/devsandbox/watchsrv.go

@@ -32,6 +32,9 @@ var DefaultLayersDir = "/layers"
 // DefaultAppDir is the default build dir
 var DefaultAppDir = utils.EnvOrDefault("CNB_APP_DIR", "/app")
 
+// DefaultAppLogDir is the saas default log dir
+var DefaultAppLogDir = utils.EnvOrDefault("CNB_APP_LOG_DIR", "/v3logs")
+
 // AppReloadEvent 事件
 type AppReloadEvent struct {
 	ID string

cnb-builder-shim/internal/devsandbox/webserver/serializer.go

@@ -0,0 +1,24 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making
+ * 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available.
+ * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ *     http://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * We undertake not to change the open source license (MIT license) applicable
+ * to the current version of the project delivered to anyone in the future.
+ */
+
+package webserver
+
+// LogQueryParams : log query params
+type LogQueryParams struct {
+	Lines int `form:"lines" binding:"omitempty,gte=1,lte=200"`
+}

cnb-builder-shim/internal/devsandbox/webserver/server.go

@@ -74,6 +74,7 @@ func New(lg *logr.Logger) (*WebServer, error) {
 	mgr := service.NewDeployManager()
 	r.POST("/deploys", DeployHandler(s, mgr))
 	r.GET("/deploys/:deployID/results", ResultHandler(mgr))
+	r.GET("/app_logs", AppLogHandler())
 
 	return s, nil
 }
@@ -196,4 +197,29 @@ func ResultHandler(svc service.DeployServiceHandler) gin.HandlerFunc {
 	}
 }
 
+// AppLogHandler 获取 app 日志
+func AppLogHandler() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		var queryParams LogQueryParams
+		if err := c.ShouldBindQuery(&queryParams); err != nil {
+			// 验证失败
+			c.JSON(http.StatusBadRequest, gin.H{
+				"message": "查询参数无效，lines 必须是 1 到 200 之间的整数",
+			})
+			return
+		}
+		// 如果没有提供 lines 参数，则设置默认值 100
+		if queryParams.Lines == 0 {
+			queryParams.Lines = 100
+		}
+		// 读取日志
+		appLogPath := path.Join(devsandbox.DefaultAppDir, devsandbox.DefaultAppLogDir)
+		logs, err := service.GetAppLogs(appLogPath, queryParams.Lines)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"message": fmt.Sprintf("get app log error: %s", err.Error())})
+		}
+		c.JSON(http.StatusOK, gin.H{"log": logs})
+	}
+}
+
 var _ devsandbox.DevWatchServer = (*WebServer)(nil)

cnb-builder-shim/internal/devsandbox/webserver/service/app_logs.go

@@ -0,0 +1,104 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making
+ * 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available.
+ * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ *     http://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * We undertake not to change the open source license (MIT license) applicable
+ * to the current version of the project delivered to anyone in the future.
+ */
+
+package service
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+func GetAppLogs(logPath string, lines int) (map[string][]string, error) {
+	logs := make(map[string][]string)
+	logFiles, err := getLatestLogFiles(logPath)
+	if err != nil {
+		return nil, err
+	}
+	for logType, files := range logFiles {
+		for _, file := range files {
+			lastXLines, err := getLastLines(logPath, file.Name(), lines)
+			if err != nil {
+				continue
+			}
+			logs[logType] = append(lastXLines, logs[logType]...)
+			if len(lastXLines) >= lines {
+				break
+			}
+		}
+	}
+	return logs, nil
+}
+
+// getLatestLogFiles 按日志类型分类，并且获取最新的日志文件列表
+// 日志文件名称需要符合格式：{{process_type}}-{{random_str}}-{{log_type}}.log
+func getLatestLogFiles(logDir string) (map[string][]os.FileInfo, error) {
+	logFiles := make(map[string][]os.FileInfo)
+	err := filepath.Walk(logDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// 过滤非日志文件, 并且根据文件名称将日志进行分类
+		if !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".log") {
+			logName := strings.TrimSuffix(strings.ToLower(info.Name()), filepath.Ext(info.Name()))
+			lastDashIndex := strings.LastIndex(logName, "-")
+			if lastDashIndex == -1 {
+				return fmt.Errorf("invalid log file name: %s", info.Name())
+			}
+			logType := logName[lastDashIndex+1:]
+			logFiles[logType] = append(logFiles[logType], info)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	// 按日志类型进行时间排序
+	for _, files := range logFiles {
+		sort.Slice(files, func(i, j int) bool {
+			return files[i].ModTime().After(files[j].ModTime())
+		})
+	}
+	return logFiles, nil
+}
+
+// getLastLines returns the last lines of the log file
+func getLastLines(logPath string, fileName string, lines int) ([]string, error) {
+	filePath := filepath.Join(logPath, fileName)
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	var logs []string
+	scanner := bufio.NewScanner(file)
+	// 全量读取日志文件,日志文件都有进行分片,对于内存压力应该还好
+	for scanner.Scan() {
+		logs = append(logs, scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	if len(logs) > lines {
+		logs = logs[len(logs)-lines:]
+	}
+	return logs, nil
+}

cnb-builder-shim/internal/devsandbox/webserver/service/deploy_test.go

@@ -34,7 +34,7 @@ var _ = Describe("Test DeployManager", func() {
 	var m *DeployManager
 	var tmpAppDir string
 
-	testSrcFilePath := filepath.Join("testdata", "templates", "django-helloworld")
+	testSrcFilePath := filepath.Join("testdata", "templates", "helloworld")
 	oldAppDir := devsandbox.DefaultAppDir
 
 	BeforeEach(func() {
@@ -55,12 +55,13 @@ var _ = Describe("Test DeployManager", func() {
 
 	Describe("Test deploy", func() {
 		It("test deploy", func() {
-			result, _ := m.Deploy(testSrcFilePath)
+			result, err := m.Deploy(testSrcFilePath)
+			Expect(err).To(BeNil())
 
 			Expect(len(result.DeployID)).To(Equal(32))
 			Expect(result.Status).To(Equal(devsandbox.ReloadProcessing))
 
-			_, err := os.Stat(path.Join(devsandbox.DefaultAppDir, "Procfile"))
+			_, err = os.Stat(path.Join(devsandbox.DefaultAppDir, "Procfile"))
 			Expect(err).To(BeNil())
 
 			// 验证隐藏目录不会被覆盖(删除)
@@ -102,3 +103,46 @@ var _ = Describe("Test DeployManager", func() {
 		})
 	})
 })
+
+var _ = Describe("Test GetAppLogs", func() {
+	var err error
+	var logPath string
+	var celeryLogPath string
+	var mysqlLogPath string
+	var newCeleryLogPath string
+	BeforeEach(func() {
+		logPath, err = os.MkdirTemp("", "log_test")
+		Expect(err).To(BeNil())
+		celeryLogPath = filepath.Join(logPath, "celery-test-celery.log")
+		mysqlLogPath = filepath.Join(logPath, "web-test-mysql.log")
+		newCeleryLogPath = filepath.Join(logPath, "celery-test-new-celery.log")
+		logContent1 := "value1\nvalue2\n"
+		logContent2 := "value3\nvalue4\n"
+		err := os.WriteFile(celeryLogPath, []byte(logContent1), 0644)
+		Expect(err).To(BeNil())
+		err = os.WriteFile(newCeleryLogPath, []byte(logContent2), 0644)
+		err = os.WriteFile(mysqlLogPath, []byte(logContent2), 0644)
+		Expect(err).To(BeNil())
+	})
+	AfterEach(func() {
+		Expect(os.RemoveAll(logPath)).To(BeNil())
+	})
+	Describe("Test GetAppLogs", func() {
+		It("test lines < logs", func() {
+			logs, err := GetAppLogs(logPath, 1)
+			Expect(err).To(BeNil())
+			Expect(len(logs["celery"])).To(Equal(1))
+			Expect(logs["celery"]).To(Equal([]string{"value4"}))
+			Expect(len(logs["mysql"])).To(Equal(1))
+			Expect(logs["mysql"]).To(Equal([]string{"value4"}))
+		})
+		It("test lines > logs", func() {
+			logs, err := GetAppLogs(logPath, 5)
+			Expect(err).To(BeNil())
+			Expect(len(logs["celery"])).To(Equal(4))
+			Expect(logs["celery"]).To(Equal([]string{"value1", "value2", "value3", "value4"}))
+			Expect(len(logs["mysql"])).To(Equal(2))
+			Expect(logs["mysql"]).To(Equal([]string{"value3", "value4"}))
+		})
+	})
+})

cnb-builder-shim/internal/devsandbox/webserver/service/testdata/templates/helloworld/app_desc.yaml

@@ -0,0 +1,8 @@
+spec_version: 2
+module:
+  language: Python
+  scripts:
+    pre_release_hook: "python manage.py migrate --no-input"
+  processes:
+    web:
+      command: python manage.py runserver 0.0.0.0:8080