Merge branch 'develop' of https://github.com/astaxie/beego into develop

� Conflicts:
�	go.mod
�	go.sum

admin.go

@@ -24,6 +24,8 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	"github.com/astaxie/beego/grace"
 	"github.com/astaxie/beego/logs"
 	"github.com/astaxie/beego/toolbox"
@@ -55,12 +57,14 @@ func init() {
 	beeAdminApp = &adminApp{
 		routers: make(map[string]http.HandlerFunc),
 	}
+	// keep in mind that all data should be html escaped to avoid XSS attack
 	beeAdminApp.Route("/", adminIndex)
 	beeAdminApp.Route("/qps", qpsIndex)
 	beeAdminApp.Route("/prof", profIndex)
 	beeAdminApp.Route("/healthcheck", healthcheck)
 	beeAdminApp.Route("/task", taskStatus)
 	beeAdminApp.Route("/listconf", listConf)
+	beeAdminApp.Route("/metrics", promhttp.Handler().ServeHTTP)
 	FilterMonitorFunc = func(string, string, time.Duration, string, int) bool { return true }
 }
 
@@ -105,8 +109,8 @@ func listConf(rw http.ResponseWriter, r *http.Request) {
 	case "conf":
 		m := make(M)
 		list("BConfig", BConfig, m)
-		m["AppConfigPath"] = appConfigPath
-		m["AppConfigProvider"] = appConfigProvider
+		m["AppConfigPath"] = template.HTMLEscapeString(appConfigPath)
+		m["AppConfigProvider"] = template.HTMLEscapeString(appConfigProvider)
 		tmpl := template.Must(template.New("dashboard").Parse(dashboardTpl))
 		tmpl = template.Must(tmpl.Parse(configTpl))
 		tmpl = template.Must(tmpl.Parse(defaultScriptsTpl))
@@ -151,8 +155,9 @@ func listConf(rw http.ResponseWriter, r *http.Request) {
 					resultList := new([][]string)
 					for _, f := range bf {
 						var result = []string{
-							f.pattern,
-							utils.GetFuncName(f.filterFunc),
+							// void xss
+							template.HTMLEscapeString(f.pattern),
+							template.HTMLEscapeString(utils.GetFuncName(f.filterFunc)),
 						}
 						*resultList = append(*resultList, result)
 					}
@@ -207,8 +212,8 @@ func PrintTree() M {
 
 		printTree(resultList, t)
 
-		methods = append(methods, method)
-		methodsData[method] = resultList
+		methods = append(methods, template.HTMLEscapeString(method))
+		methodsData[template.HTMLEscapeString(method)] = resultList
 	}
 
 	content["Data"] = methodsData
@@ -227,21 +232,21 @@ func printTree(resultList *[][]string, t *Tree) {
 		if v, ok := l.runObject.(*ControllerInfo); ok {
 			if v.routerType == routerTypeBeego {
 				var result = []string{
-					v.pattern,
-					fmt.Sprintf("%s", v.methods),
-					v.controllerType.String(),
+					template.HTMLEscapeString(v.pattern),
+					template.HTMLEscapeString(fmt.Sprintf("%s", v.methods)),
+					template.HTMLEscapeString(v.controllerType.String()),
 				}
 				*resultList = append(*resultList, result)
 			} else if v.routerType == routerTypeRESTFul {
 				var result = []string{
-					v.pattern,
-					fmt.Sprintf("%s", v.methods),
+					template.HTMLEscapeString(v.pattern),
+					template.HTMLEscapeString(fmt.Sprintf("%s", v.methods)),
 					"",
 				}
 				*resultList = append(*resultList, result)
 			} else if v.routerType == routerTypeHandler {
 				var result = []string{
-					v.pattern,
+					template.HTMLEscapeString(v.pattern),
 					"",
 					"",
 				}
@@ -266,7 +271,7 @@ func profIndex(rw http.ResponseWriter, r *http.Request) {
 		result bytes.Buffer
 	)
 	toolbox.ProcessInput(command, &result)
-	data["Content"] = result.String()
+	data["Content"] = template.HTMLEscapeString(result.String())
 
 	if format == "json" && command == "gc summary" {
 		dataJSON, err := json.Marshal(data)
@@ -280,7 +285,7 @@ func profIndex(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	data["Title"] = command
+	data["Title"] = template.HTMLEscapeString(command)
 	defaultTpl := defaultScriptsTpl
 	if command == "gc summary" {
 		defaultTpl = gcAjaxTpl
@@ -304,13 +309,13 @@ func healthcheck(rw http.ResponseWriter, _ *http.Request) {
 		if err := h.Check(); err != nil {
 			result = []string{
 				"error",
-				name,
-				err.Error(),
+				template.HTMLEscapeString(name),
+				template.HTMLEscapeString(err.Error()),
 			}
 		} else {
 			result = []string{
 				"success",
-				name,
+				template.HTMLEscapeString(name),
 				"OK",
 			}
 		}
@@ -334,11 +339,11 @@ func taskStatus(rw http.ResponseWriter, req *http.Request) {
 	if taskname != "" {
 		if t, ok := toolbox.AdminTaskList[taskname]; ok {
 			if err := t.Run(); err != nil {
-				data["Message"] = []string{"error", fmt.Sprintf("%s", err)}
+				data["Message"] = []string{"error", template.HTMLEscapeString(fmt.Sprintf("%s", err))}
 			}
-			data["Message"] = []string{"success", fmt.Sprintf("%s run success,Now the Status is <br>%s", taskname, t.GetStatus())}
+			data["Message"] = []string{"success", template.HTMLEscapeString(fmt.Sprintf("%s run success,Now the Status is <br>%s", taskname, t.GetStatus()))}
 		} else {
-			data["Message"] = []string{"warning", fmt.Sprintf("there's no task which named: %s", taskname)}
+			data["Message"] = []string{"warning", template.HTMLEscapeString(fmt.Sprintf("there's no task which named: %s", taskname))}
 		}
 	}
 
@@ -354,10 +359,10 @@ func taskStatus(rw http.ResponseWriter, req *http.Request) {
 	}
 	for tname, tk := range toolbox.AdminTaskList {
 		result := []string{
-			tname,
-			tk.GetSpec(),
-			tk.GetStatus(),
-			tk.GetPrev().String(),
+			template.HTMLEscapeString(tname),
+			template.HTMLEscapeString(tk.GetSpec()),
+			template.HTMLEscapeString(tk.GetStatus()),
+			template.HTMLEscapeString(tk.GetPrev().String()),
 		}
 		*resultList = append(*resultList, result)
 	}

app.go

@@ -130,7 +130,6 @@ func (app *App) Run(mws ...MiddleWare) {
 					if err := server.ListenAndServeMutualTLS(BConfig.Listen.HTTPSCertFile, BConfig.Listen.HTTPSKeyFile, BConfig.Listen.TrustCaFile); err != nil {
 						logs.Critical("ListenAndServeTLS: ", err, fmt.Sprintf("%d", os.Getpid()))
 						time.Sleep(100 * time.Microsecond)
-						endRunning <- true
 					}
 				} else {
 					if BConfig.Listen.AutoTLS {
@@ -145,9 +144,9 @@ func (app *App) Run(mws ...MiddleWare) {
 					if err := server.ListenAndServeTLS(BConfig.Listen.HTTPSCertFile, BConfig.Listen.HTTPSKeyFile); err != nil {
 						logs.Critical("ListenAndServeTLS: ", err, fmt.Sprintf("%d", os.Getpid()))
 						time.Sleep(100 * time.Microsecond)
-						endRunning <- true
 					}
 				}
+				endRunning <- true
 			}()
 		}
 		if BConfig.Listen.EnableHTTP {
@@ -161,8 +160,8 @@ func (app *App) Run(mws ...MiddleWare) {
 				if err := server.ListenAndServe(); err != nil {
 					logs.Critical("ListenAndServe: ", err, fmt.Sprintf("%d", os.Getpid()))
 					time.Sleep(100 * time.Microsecond)
-					endRunning <- true
 				}
+				endRunning <- true
 			}()
 		}
 		<-endRunning

build_info.go

@@ -0,0 +1,27 @@
+// Copyright 2020 astaxie
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package beego
+
+var (
+	BuildVersion string
+	BuildGitRevision string
+	BuildStatus string
+	BuildTag string
+	BuildTime string
+
+	GoVersion string
+
+	GitBranch string
+)

cache/redis/redis.go

@@ -55,6 +55,9 @@ type Cache struct {
 	key      string
 	password string
 	maxIdle  int
+
+	//the timeout to a value less than the redis server's timeout.
+	timeout  time.Duration
 }
 
 // NewRedisCache create new redis cache with default collection name.
@@ -137,12 +140,12 @@ func (rc *Cache) Decr(key string) error {
 
 // ClearAll clean all cache in redis. delete this redis collection.
 func (rc *Cache) ClearAll() error {
-	c := rc.p.Get()
-	defer c.Close()
-	cachedKeys, err := redis.Strings(c.Do("KEYS", rc.key+":*"))
+	cachedKeys, err := rc.Scan(rc.key + ":*")
 	if err != nil {
 		return err
 	}
+	c := rc.p.Get()
+	defer c.Close()
 	for _, str := range cachedKeys {
 		if _, err = c.Do("DEL", str); err != nil {
 			return err
@@ -151,6 +154,35 @@ func (rc *Cache) ClearAll() error {
 	return err
 }
 
+// Scan scan all keys matching the pattern. a better choice than `keys`
+func (rc *Cache) Scan(pattern string) (keys []string, err error) {
+	c := rc.p.Get()
+	defer c.Close()
+	var (
+		cursor uint64 = 0 // start
+		result []interface{}
+		list   []string
+	)
+	for {
+		result, err = redis.Values(c.Do("SCAN", cursor, "MATCH", pattern, "COUNT", 1024))
+		if err != nil {
+			return
+		}
+		list, err = redis.Strings(result[1], nil)
+		if err != nil {
+			return
+		}
+		keys = append(keys, list...)
+		cursor, err = redis.Uint64(result[0], nil)
+		if err != nil {
+			return
+		}
+		if cursor == 0 { // over
+			return
+		}
+	}
+}
+
 // StartAndGC start redis cache adapter.
 // config is like {"key":"collection key","conn":"connection info","dbNum":"0"}
 // the cache item in redis are stored forever,
@@ -182,12 +214,21 @@ func (rc *Cache) StartAndGC(config string) error {
 	if _, ok := cf["maxIdle"]; !ok {
 		cf["maxIdle"] = "3"
 	}
+	if _, ok := cf["timeout"]; !ok {
+		cf["timeout"] = "180s"
+	}
 	rc.key = cf["key"]
 	rc.conninfo = cf["conn"]
 	rc.dbNum, _ = strconv.Atoi(cf["dbNum"])
 	rc.password = cf["password"]
 	rc.maxIdle, _ = strconv.Atoi(cf["maxIdle"])
 
+	if v, err := time.ParseDuration(cf["timeout"]); err == nil {
+		rc.timeout = v
+	} else {
+		rc.timeout = 180 * time.Second
+	}
+
 	rc.connectInit()
 
 	c := rc.p.Get()
@@ -221,7 +262,7 @@ func (rc *Cache) connectInit() {
 	// initialize a new pool
 	rc.p = &redis.Pool{
 		MaxIdle:     rc.maxIdle,
-		IdleTimeout: 180 * time.Second,
+		IdleTimeout: rc.timeout,
 		Dial:        dialFunc,
 	}
 }

cache/redis/redis_test.go

@@ -15,6 +15,7 @@
 package redis
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -104,3 +105,40 @@ func TestRedisCache(t *testing.T) {
 		t.Error("clear all err")
 	}
 }
+
+func TestCache_Scan(t *testing.T) {
+	timeoutDuration := 10 * time.Second
+	// init
+	bm, err := cache.NewCache("redis", `{"conn": "127.0.0.1:6379"}`)
+	if err != nil {
+		t.Error("init err")
+	}
+	// insert all
+	for i := 0; i < 10000; i++ {
+		if err = bm.Put(fmt.Sprintf("astaxie%d", i), fmt.Sprintf("author%d", i), timeoutDuration); err != nil {
+			t.Error("set Error", err)
+		}
+	}
+	// scan all for the first time
+	keys, err := bm.(*Cache).Scan(DefaultKey + ":*")
+	if err != nil {
+		t.Error("scan Error", err)
+	}
+	if len(keys) != 10000 {
+		t.Error("scan all err")
+	}
+
+	// clear all
+	if err = bm.ClearAll(); err != nil {
+		t.Error("clear all err")
+	}
+
+	// scan all for the second time
+	keys, err = bm.(*Cache).Scan(DefaultKey + ":*")
+	if err != nil {
+		t.Error("scan Error", err)
+	}
+	if len(keys) != 0 {
+		t.Error("scan all err")
+	}
+}

config.go

@@ -131,6 +131,8 @@ var (
 	appConfigPath string
 	// appConfigProvider is the provider for the config, default is ini
 	appConfigProvider = "ini"
+	// WorkPath is the absolute path to project root directory
+	WorkPath string
 )
 
 func init() {
@@ -139,15 +141,15 @@ func init() {
 	if AppPath, err = filepath.Abs(filepath.Dir(os.Args[0])); err != nil {
 		panic(err)
 	}
-	workPath, err := os.Getwd()
+	WorkPath, err = os.Getwd()
 	if err != nil {
 		panic(err)
 	}
 	var filename = "app.conf"
 	if os.Getenv("BEEGO_RUNMODE") != "" {
 		filename = os.Getenv("BEEGO_RUNMODE") + ".app.conf"
 	}
-	appConfigPath = filepath.Join(workPath, "conf", filename)
+	appConfigPath = filepath.Join(WorkPath, "conf", filename)
 	if !utils.FileExists(appConfigPath) {
 		appConfigPath = filepath.Join(AppPath, "conf", filename)
 		if !utils.FileExists(appConfigPath) {
@@ -420,9 +422,9 @@ func newAppConfig(appConfigProvider, appConfigPath string) (*beegoAppConfig, err
 
 func (b *beegoAppConfig) Set(key, val string) error {
 	if err := b.innerConfig.Set(BConfig.RunMode+"::"+key, val); err != nil {
-		return err
+		return b.innerConfig.Set(key, val)
 	}
-	return b.innerConfig.Set(key, val)
+	return nil
 }
 
 func (b *beegoAppConfig) String(key string) string {