From c54bc8467b70fb578c38116b9d853002c2f2616f Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 08:29:05 +0800 Subject: [PATCH 01/21] chore: refactor log --- api/filter/logging.go | 106 +++++++------- api/filter/recover.go | 15 +- api/go.mod | 1 + api/go.sum | 2 + api/internal/handler/route/route.go | 4 +- api/log/log.go | 207 ++++++++-------------------- api/log/zap.go | 45 ++++++ api/main.go | 14 +- 8 files changed, 169 insertions(+), 225 deletions(-) create mode 100644 api/log/zap.go diff --git a/api/filter/logging.go b/api/filter/logging.go index a62aef5c7f..de4acda2de 100644 --- a/api/filter/logging.go +++ b/api/filter/logging.go @@ -18,65 +18,61 @@ package filter import ( "bytes" - "io/ioutil" - "time" "github.com/gin-gonic/gin" - "github.com/shiningrush/droplet/log" - "github.com/sirupsen/logrus" ) -func RequestLogHandler() gin.HandlerFunc { - return func(c *gin.Context) { - start, host, remoteIP, path, method := time.Now(), c.Request.Host, c.ClientIP(), c.Request.URL.Path, c.Request.Method - var val interface{} - if method == "GET" { - val = c.Request.URL.Query() - } else { - val, _ = c.GetRawData() - - // set RequestBody back - c.Request.Body = ioutil.NopCloser(bytes.NewReader(val.([]byte))) - } - c.Set("requestBody", val) - uuid, _ := c.Get("X-Request-Id") - - param, _ := c.Get("requestBody") - - switch paramType := param.(type) { - case []byte: - param = string(param.([]byte)) - log.Infof("type of param: %#v", paramType) - default: - } - - blw := &bodyLogWriter{body: bytes.NewBufferString(""), ResponseWriter: c.Writer} - c.Writer = blw - c.Next() - latency := time.Since(start) / 1000000 - statusCode := c.Writer.Status() - respBody := blw.body.String() - if uuid == "" { - uuid = c.Writer.Header().Get("X-Request-Id") - } - var errs []string - for _, err := range c.Errors { - errs = append(errs, err.Error()) - } - logger.WithFields(logrus.Fields{ - "requestId": uuid, - "latency": latency, - "remoteIp": remoteIP, - "method": method, - "path": path, - "statusCode": statusCode, - "host": host, - "params": param, - "respBody": respBody, - "errMsg": errs, - }).Info("") - } -} +//func RequestLogHandler() gin.HandlerFunc { +// //return func(c *gin.Context) { +// // start, host, remoteIP, path, method := time.Now(), c.Request.Host, c.ClientIP(), c.Request.URL.Path, c.Request.Method +// // var val interface{} +// // if method == "GET" { +// // val = c.Request.URL.Query() +// // } else { +// // val, _ = c.GetRawData() +// // +// // // set RequestBody back +// // c.Request.Body = ioutil.NopCloser(bytes.NewReader(val.([]byte))) +// // } +// // c.Set("requestBody", val) +// // uuid, _ := c.Get("X-Request-Id") +// // +// // param, _ := c.Get("requestBody") +// // +// // switch paramType := param.(type) { +// // case []byte: +// // param = string(param.([]byte)) +// // log.Infof("type of param: %#v", paramType) +// // default: +// // } +// // +// // blw := &bodyLogWriter{body: bytes.NewBufferString(""), ResponseWriter: c.Writer} +// // c.Writer = blw +// // c.Next() +// // latency := time.Since(start) / 1000000 +// // statusCode := c.Writer.Status() +// // respBody := blw.body.String() +// // if uuid == "" { +// // uuid = c.Writer.Header().Get("X-Request-Id") +// // } +// // var errs []string +// // for _, err := range c.Errors { +// // errs = append(errs, err.Error()) +// // } +// // logger.With(logrus.Fields{ +// // "requestId": uuid, +// // "latency": latency, +// // "remoteIp": remoteIP, +// // "method": method, +// // "path": path, +// // "statusCode": statusCode, +// // "host": host, +// // "params": param, +// // "respBody": respBody, +// // "errMsg": errs, +// // }).Info("") +// //} +//} type bodyLogWriter struct { gin.ResponseWriter diff --git a/api/filter/recover.go b/api/filter/recover.go index a47e270d88..77e44a3311 100644 --- a/api/filter/recover.go +++ b/api/filter/recover.go @@ -26,11 +26,9 @@ import ( "github.com/apisix/manager-api/log" "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" ) var ( - logger = log.GetLogger() dunno = []byte("???") centerDot = []byte("·") dot = []byte(".") @@ -41,12 +39,13 @@ func RecoverHandler() gin.HandlerFunc { return func(c *gin.Context) { defer func() { if err := recover(); err != nil { - uuid := c.Writer.Header().Get("X-Request-Id") - logger.WithFields(logrus.Fields{ - "uuid": uuid, - }) + fmt.Println("err;", err) + //uuid := c.Writer.Header().Get("X-Request-Id") stack := stack(3) - logger.Errorf("[Recovery] %s panic recovered:\n\n%s\n%s", timeFormat(time.Now()), err, stack) + fmt.Printf("[Recovery] %s panic recovered:\n\n%s\n%s", timeFormat(time.Now()), err, stack) + + //log.With(zap.String("uuid", uuid)) + log.Errorf("[Recovery] %s panic recovered:\n\n%s\n%s", timeFormat(time.Now()), err, stack) c.AbortWithStatus(http.StatusInternalServerError) } }() @@ -58,7 +57,7 @@ func WrapGo(f func(...interface{}), args ...interface{}) { defer func() { if err := recover(); err != nil { stack := stack(3) - logger.Errorf("[Recovery] %s panic recovered:\n\n%s\n%s", timeFormat(time.Now()), err, stack) + log.Errorf("[Recovery] %s panic recovered:\n\n%s\n%s", timeFormat(time.Now()), err, stack) } }() f(args...) diff --git a/api/go.mod b/api/go.mod index 3f41ce6d42..017a572059 100644 --- a/api/go.mod +++ b/api/go.mod @@ -17,6 +17,7 @@ require ( github.com/gin-gonic/gin v1.6.3 github.com/gogo/protobuf v1.3.1 // indirect github.com/google/uuid v1.1.2 // indirect + github.com/natefinch/lumberjack v2.0.0+incompatible github.com/satori/go.uuid v1.2.0 github.com/shiningrush/droplet v0.2.1 github.com/shiningrush/droplet/wrapper/gin v0.2.0 diff --git a/api/go.sum b/api/go.sum index 7dcfbc192d..2eae4fad02 100644 --- a/api/go.sum +++ b/api/go.sum @@ -97,6 +97,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= +github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/api/internal/handler/route/route.go b/api/internal/handler/route/route.go index e9d1d80bd6..07e6193c31 100644 --- a/api/internal/handler/route/route.go +++ b/api/internal/handler/route/route.go @@ -28,7 +28,6 @@ import ( "github.com/gin-gonic/gin" "github.com/shiningrush/droplet" "github.com/shiningrush/droplet/data" - "github.com/shiningrush/droplet/log" "github.com/shiningrush/droplet/wrapper" wgin "github.com/shiningrush/droplet/wrapper/gin" @@ -38,6 +37,7 @@ import ( "github.com/apisix/manager-api/internal/handler" "github.com/apisix/manager-api/internal/utils" "github.com/apisix/manager-api/internal/utils/consts" + "github.com/apisix/manager-api/log" ) type Handler struct { @@ -328,7 +328,7 @@ func (h *Handler) BatchDelete(c droplet.Context) (interface{}, error) { //delete stored script if err := h.scriptStore.BatchDelete(c.Context(), strings.Split(input.IDs, ",")); err != nil { //try again - log.Warnf("try to delete script %s again", input.IDs) + log.Warn("try to delete script %s again", input.IDs) if err := h.scriptStore.BatchDelete(c.Context(), strings.Split(input.IDs, ",")); err != nil { return nil, nil } diff --git a/api/log/log.go b/api/log/log.go index b9e68603c8..3c53a96375 100644 --- a/api/log/log.go +++ b/api/log/log.go @@ -16,188 +16,93 @@ */ package log -import ( - "bufio" - "fmt" - "os" - "runtime" - "strings" - - "github.com/apisix/manager-api/conf" - "github.com/sirupsen/logrus" +var ( + DefLogger Interface = emptyLog{} ) -var logEntry *logrus.Entry - -type DefLogger struct { +type emptyLog struct { } -func getLogrusFields(entry *logrus.Entry, fs []interface{}) *logrus.Entry { - if len(fs) == 0 { - return entry - } - - fd := logrus.Fields{} - for i := 0; i < len(fs); i += 2 { - fd[fs[i].(string)] = fs[i+1] - } - return entry.WithFields(fd) +type Interface interface { + Debug(msg string, fields ...interface{}) + Debugf(msg string, args ...interface{}) + Info(msg string, fields ...interface{}) + Infof(msg string, args ...interface{}) + Warn(msg string, fields ...interface{}) + Warnf(msg string, args ...interface{}) + Error(msg string, fields ...interface{}) + Errorf(msg string, args ...interface{}) + Fatal(msg string, fields ...interface{}) + Fatalf(msg string, args ...interface{}) } -func (e DefLogger) Debug(msg string, fields ...interface{}) { - getLogrusFields(logEntry, fields).Debug(msg) +func (e emptyLog) Debug(msg string, fields ...interface{}) { + getZapFields(logger, fields).Debug(msg) } -func (e DefLogger) Debugf(msg string, args ...interface{}) { - logEntry.Debugf(msg, args...) +func (e emptyLog) Debugf(msg string, args ...interface{}) { + logger.Debugf(msg, args...) } -func (e DefLogger) Info(msg string, fields ...interface{}) { - getLogrusFields(logEntry, fields).Info(msg) +func (e emptyLog) Info(msg string, fields ...interface{}) { + getZapFields(logger, fields).Info(msg) } -func (e DefLogger) Infof(msg string, args ...interface{}) { - logEntry.Infof(msg, args...) +func (e emptyLog) Infof(msg string, args ...interface{}) { + logger.Infof(msg, args...) } -func (e DefLogger) Warn(msg string, fields ...interface{}) { - getLogrusFields(logEntry, fields).Warn(msg) +func (e emptyLog) Warn(msg string, fields ...interface{}) { + getZapFields(logger, fields).Warn(msg) } -func (e DefLogger) Warnf(msg string, args ...interface{}) { - logEntry.Warnf(msg, args...) +func (e emptyLog) Warnf(msg string, args ...interface{}) { + logger.Warnf(msg, args...) } -func (e DefLogger) Error(msg string, fields ...interface{}) { - getLogrusFields(logEntry, fields).Error(msg) +func (e emptyLog) Error(msg string, fields ...interface{}) { + getZapFields(logger, fields).Error(msg) } -func (e DefLogger) Errorf(msg string, args ...interface{}) { - logEntry.Errorf(msg, args...) +func (e emptyLog) Errorf(msg string, args ...interface{}) { + logger.Errorf(msg, args...) } -func (e DefLogger) Fatal(msg string, fields ...interface{}) { - getLogrusFields(logEntry, fields).Fatal(msg) +func (e emptyLog) Fatal(msg string, fields ...interface{}) { + getZapFields(logger, fields).Fatal(msg) } -func (e DefLogger) Fatalf(msg string, args ...interface{}) { - logEntry.Fatalf(msg, args...) +func (e emptyLog) Fatalf(msg string, args ...interface{}) { + logger.Fatalf(msg, args...) } -func GetLogger() *logrus.Entry { - if logEntry == nil { - var log = logrus.New() - setNull(log) - log.SetLevel(logrus.DebugLevel) - if conf.ENV != conf.EnvLOCAL { - log.SetLevel(logrus.ErrorLevel) - } - log.SetFormatter(&logrus.JSONFormatter{}) - logEntry = log.WithFields(logrus.Fields{ - "app": "manager-api", - }) - if hook, err := createHook(); err == nil { - log.AddHook(hook) - } - } - return logEntry +func Debug(msg string, fields ...interface{}) { + DefLogger.Debug(msg, fields...) } - -func setNull(log *logrus.Logger) { - src, err := os.OpenFile(os.DevNull, os.O_APPEND|os.O_WRONLY, os.ModeAppend) - if err != nil { - fmt.Println("err", err) - } - writer := bufio.NewWriter(src) - log.SetOutput(writer) +func Debugf(msg string, args ...interface{}) { + DefLogger.Debugf(msg, args...) } - -type Hook struct { - Formatter func(file, function string, line int) string +func Info(msg string, fields ...interface{}) { + DefLogger.Info(msg, fields...) } - -func createHook() (*Hook, error) { - return &Hook{ - func(file, function string, line int) string { - return fmt.Sprintf("%s:%d", file, line) - }, - }, nil +func Infof(msg string, args ...interface{}) { + DefLogger.Infof(msg, args...) } - -func (hook *Hook) Fire(entry *logrus.Entry) error { - str := hook.Formatter(findCaller(5)) - en := entry.WithField("line", str) - en.Level = entry.Level - en.Message = entry.Message - en.Time = entry.Time - line, err := en.String() - if err != nil { - fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err) - return err - } - switch en.Level { - case logrus.PanicLevel: - fmt.Print(line) - return nil - case logrus.FatalLevel: - fmt.Print(line) - return nil - case logrus.ErrorLevel: - fmt.Print(line) - return nil - case logrus.WarnLevel: - fmt.Print(line) - return nil - case logrus.InfoLevel: - fmt.Print(line) - return nil - case logrus.DebugLevel: - fmt.Print(line) - return nil - default: - return nil - } +func Warn(msg string, fields ...interface{}) { + DefLogger.Warn(msg, fields...) } - -func (hook *Hook) Levels() []logrus.Level { - return logrus.AllLevels +func Warnf(msg string, args ...interface{}) { + DefLogger.Warnf(msg, args...) } - -func findCaller(skip int) (string, string, int) { - var ( - pc uintptr - file string - function string - line int - ) - for i := 0; i < 10; i++ { - pc, file, line = getCaller(skip + i) - if !strings.HasPrefix(file, "logrus") { - break - } - } - if pc != 0 { - frames := runtime.CallersFrames([]uintptr{pc}) - frame, _ := frames.Next() - function = frame.Function - } - return file, function, line +func Error(msg string, fields ...interface{}) { + DefLogger.Error(msg, fields...) } - -func getCaller(skip int) (uintptr, string, int) { - pc, file, line, ok := runtime.Caller(skip) - if !ok { - return 0, "", 0 - } - n := 0 - for i := len(file) - 1; i > 0; i-- { - if file[i] == '/' { - n += 1 - if n >= 2 { - file = file[i+1:] - break - } - } - } - return pc, file, line +func Errorf(msg string, args ...interface{}) { + DefLogger.Errorf(msg, args...) +} +func Fatal(msg string, fields ...interface{}) { + DefLogger.Fatal(msg, fields...) +} +func Fatalf(msg string, args ...interface{}) { + DefLogger.Fatalf(msg, args...) } diff --git a/api/log/zap.go b/api/log/zap.go new file mode 100644 index 0000000000..87c90ec02d --- /dev/null +++ b/api/log/zap.go @@ -0,0 +1,45 @@ +package log + +import ( + "github.com/natefinch/lumberjack" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var logger *zap.SugaredLogger + +func init() { + writeSyncer := getLogWriter() + encoder := getEncoder() + core := zapcore.NewCore(encoder, writeSyncer, zapcore.DebugLevel) + + zapLogger := zap.New(core, zap.AddCaller()) + + logger = zapLogger.Sugar() +} + +func getEncoder() zapcore.Encoder { + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder + return zapcore.NewConsoleEncoder(encoderConfig) +} + +func getLogWriter() zapcore.WriteSyncer { + lumberJackLogger := &lumberjack.Logger{ + Filename: "./test.log", + MaxSize: 1, + MaxBackups: 5, + MaxAge: 30, + Compress: false, + } + return zapcore.AddSync(lumberJackLogger) +} + +func getZapFields(logger *zap.SugaredLogger, fields []interface{}) *zap.SugaredLogger { + if len(fields) == 0 { + return logger + } + + return logger.With(fields) +} diff --git a/api/main.go b/api/main.go index 1664567977..b12c77425f 100644 --- a/api/main.go +++ b/api/main.go @@ -18,29 +18,25 @@ package main import ( "fmt" - "log" "net/http" "strings" "time" - dlog "github.com/shiningrush/droplet/log" - "github.com/apisix/manager-api/conf" "github.com/apisix/manager-api/internal" "github.com/apisix/manager-api/internal/core/storage" "github.com/apisix/manager-api/internal/core/store" "github.com/apisix/manager-api/internal/utils" - alog "github.com/apisix/manager-api/log" + "github.com/apisix/manager-api/log" ) -var logger = alog.GetLogger() - func main() { - dlog.DefLogger = alog.DefLogger{} if err := storage.InitETCDClient(strings.Split(conf.ETCDEndpoints, ",")); err != nil { + log.Error("init etcd client fail: %w", err) panic(err) } if err := store.InitStores(); err != nil { + log.Error("init stores fail: %w", err) panic(err) } // routes @@ -53,10 +49,10 @@ func main() { WriteTimeout: time.Duration(5000) * time.Millisecond, } - log.Printf("The Manager API is listening on %s ", addr) + log.Infof("The Manager API is listening on %s ", addr) if err := s.ListenAndServe(); err != nil { - logger.WithError(err) + log.Error("listen and serv fail: %w", err) } utils.CloseAll() From 52a2ac961396330199b7fca82d71229b99940f55 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 13:51:00 +0800 Subject: [PATCH 02/21] fix: custom log by conf --- api/conf/conf.go | 32 ++++++++++++++++++++++++++++++++ api/conf/conf.json | 11 +++++++++++ api/go.mod | 2 ++ api/go.sum | 6 ++++++ api/log/zap.go | 46 ++++++++++++++++++++++++++++++++++------------ 5 files changed, 85 insertions(+), 12 deletions(-) diff --git a/api/conf/conf.go b/api/conf/conf.go index c12a08503b..3285918152 100644 --- a/api/conf/conf.go +++ b/api/conf/conf.go @@ -48,6 +48,11 @@ var ( ServerHost = "127.0.0.1" ServerPort = 80 ETCDEndpoints = "127.0.0.1:2379" + ErrorLogLevel = "warn" + ErrorLogPath = "./logs/error.log" + LogRotateInterval int64 = 86400 //second + LogRotateMaxSize int64 = 100000000 //byte + LogRotateMaxAge int64 = 2592000 //second ) func init() { @@ -68,6 +73,7 @@ func setConf() { if serverPort != 0 { ServerPort = serverPort } + serverHost := configuration.Get("conf.listen.host").String() if serverHost != "" { ServerHost = serverHost @@ -78,12 +84,38 @@ func setConf() { if dagLibPath != "" { DagLibPath = dagLibPath } + //etcd eTCDEndpoints := configuration.Get("conf.etcd.endpoints").String() if eTCDEndpoints != "" { ETCDEndpoints = eTCDEndpoints } + //log + logLevel := configuration.Get("conf.log.error_log.level").String() + if logLevel != "" { + ErrorLogLevel = logLevel + } + + errorLogPath := configuration.Get("conf.log.error_log.file_path").String() + if errorLogPath != "" { + ErrorLogPath = errorLogPath + } + + rotateInterval := configuration.Get("conf.log.rotate.interval").Int() + if rotateInterval > 0 { + LogRotateInterval = rotateInterval + } + + rotateMaxSize := configuration.Get("conf.log.rotate.max_size").Int() + if rotateMaxSize > 0 { + LogRotateMaxSize = rotateMaxSize + } + + rotateMaxAge := configuration.Get("conf.log.rotate.max_age").Int() + if rotateMaxAge > 0 { + LogRotateMaxAge = rotateMaxAge + } } } diff --git a/api/conf/conf.json b/api/conf/conf.json index c369afe93c..0cdecc4e68 100644 --- a/api/conf/conf.json +++ b/api/conf/conf.json @@ -10,6 +10,17 @@ "dag-lib-path": "", "etcd": { "endpoints": "127.0.0.1:2379" + }, + "log": { + "error_log": { + "level": "info", + "file_path": "./logs/error.log" + }, + "rotate": { + "interval": 20, + "max_size": 100, + "max_age": 2592000 + } } }, "authentication": { diff --git a/api/go.mod b/api/go.mod index 017a572059..178708c2c2 100644 --- a/api/go.mod +++ b/api/go.mod @@ -17,6 +17,8 @@ require ( github.com/gin-gonic/gin v1.6.3 github.com/gogo/protobuf v1.3.1 // indirect github.com/google/uuid v1.1.2 // indirect + github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible + github.com/lestrrat-go/strftime v1.0.3 // indirect github.com/natefinch/lumberjack v2.0.0+incompatible github.com/satori/go.uuid v1.2.0 github.com/shiningrush/droplet v0.2.1 diff --git a/api/go.sum b/api/go.sum index 2eae4fad02..c1e2010feb 100644 --- a/api/go.sum +++ b/api/go.sum @@ -89,6 +89,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= +github.com/lestrrat-go/strftime v1.0.3 h1:qqOPU7y+TM8Y803I8fG9c/DyKG3xH/xkng6keC1015Q= +github.com/lestrrat-go/strftime v1.0.3/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -99,6 +104,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/api/log/zap.go b/api/log/zap.go index 87c90ec02d..455822ef97 100644 --- a/api/log/zap.go +++ b/api/log/zap.go @@ -1,23 +1,39 @@ package log import ( - "github.com/natefinch/lumberjack" + "github.com/apisix/manager-api/conf" + rotatelogs "github.com/lestrrat-go/file-rotatelogs" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "time" ) var logger *zap.SugaredLogger func init() { - writeSyncer := getLogWriter() + writeSyncer := rotateWriter() encoder := getEncoder() - core := zapcore.NewCore(encoder, writeSyncer, zapcore.DebugLevel) + logLevel := getLogLevel() + core := zapcore.NewCore(encoder, writeSyncer, logLevel) zapLogger := zap.New(core, zap.AddCaller()) logger = zapLogger.Sugar() } +func getLogLevel() zapcore.LevelEnabler{ + level := zapcore.WarnLevel + switch conf.ErrorLogLevel { + case "debug": + level = zapcore.DebugLevel + case "info": + level = zapcore.InfoLevel + case "error": + level = zapcore.ErrorLevel + } + return level +} + func getEncoder() zapcore.Encoder { encoderConfig := zap.NewProductionEncoderConfig() encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder @@ -25,17 +41,23 @@ func getEncoder() zapcore.Encoder { return zapcore.NewConsoleEncoder(encoderConfig) } -func getLogWriter() zapcore.WriteSyncer { - lumberJackLogger := &lumberjack.Logger{ - Filename: "./test.log", - MaxSize: 1, - MaxBackups: 5, - MaxAge: 30, - Compress: false, - } - return zapcore.AddSync(lumberJackLogger) +func rotateWriter() zapcore.WriteSyncer { + maxAge := time.Duration(conf.LogRotateMaxAge) + maxSize := conf.LogRotateMaxSize + interval := time.Duration(conf.LogRotateInterval) + + logf, _ := rotatelogs.New( + conf.ErrorLogPath + ".%Y%m%d%H%M%S", + rotatelogs.WithMaxAge(maxAge * time.Second), + rotatelogs.WithRotationTime(interval * time.Second), + //rotatelogs.WithRotationCount(7), + rotatelogs.WithRotationSize(maxSize), + ) + + return zapcore.AddSync(logf) } + func getZapFields(logger *zap.SugaredLogger, fields []interface{}) *zap.SugaredLogger { if len(fields) == 0 { return logger From 9e66424b32c9d5642586a944e3eff2c940fcaeb9 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 14:41:00 +0800 Subject: [PATCH 03/21] feat: add error log --- api/conf/conf.go | 24 ++++++++++++------------ api/internal/core/store/query.go | 4 +++- api/internal/core/store/store.go | 17 ++++++++++++++--- api/internal/core/store/storehub.go | 3 +++ api/internal/core/store/validate.go | 5 +++++ api/log/zap.go | 9 ++++----- 6 files changed, 41 insertions(+), 21 deletions(-) diff --git a/api/conf/conf.go b/api/conf/conf.go index 3285918152..d41bf7a56a 100644 --- a/api/conf/conf.go +++ b/api/conf/conf.go @@ -41,18 +41,18 @@ const ( ) var ( - ENV string - basePath string - Schema gjson.Result - DagLibPath = "/go/manager-api/dag-to-lua/" - ServerHost = "127.0.0.1" - ServerPort = 80 - ETCDEndpoints = "127.0.0.1:2379" - ErrorLogLevel = "warn" - ErrorLogPath = "./logs/error.log" - LogRotateInterval int64 = 86400 //second - LogRotateMaxSize int64 = 100000000 //byte - LogRotateMaxAge int64 = 2592000 //second + ENV string + basePath string + Schema gjson.Result + DagLibPath = "/go/manager-api/dag-to-lua/" + ServerHost = "127.0.0.1" + ServerPort = 80 + ETCDEndpoints = "127.0.0.1:2379" + ErrorLogLevel = "warn" + ErrorLogPath = "./logs/error.log" + LogRotateInterval int64 = 86400 //second + LogRotateMaxSize int64 = 100000000 //byte + LogRotateMaxAge int64 = 2592000 //second ) func init() { diff --git a/api/internal/core/store/query.go b/api/internal/core/store/query.go index 7f59594449..20036d06ab 100644 --- a/api/internal/core/store/query.go +++ b/api/internal/core/store/query.go @@ -16,6 +16,7 @@ package store import ( "github.com/apisix/manager-api/internal/core/entity" + "github.com/apisix/manager-api/log" ) type Query struct { @@ -88,7 +89,7 @@ func NewQuery(sort *Sort, filter *Filter, pagination *Pagination) *Query { func NewSort(sortRaw []string) *Sort { if sortRaw == nil || len(sortRaw)%2 == 1 { - // Empty sort list or invalid (odd) length + log.Info("empty sort for query") return NoSort } list := []SortBy{} @@ -117,6 +118,7 @@ func NewSort(sortRaw []string) *Sort { func NewFilter(filterRaw []string) *Filter { if filterRaw == nil || len(filterRaw)%2 == 1 { + log.Info("empty filter for query") return NoFilter } list := []FilterBy{} diff --git a/api/internal/core/store/store.go b/api/internal/core/store/store.go index b285183dfb..1627e2ee9c 100644 --- a/api/internal/core/store/store.go +++ b/api/internal/core/store/store.go @@ -21,7 +21,6 @@ import ( "context" "encoding/json" "fmt" - "log" "reflect" "sort" "sync" @@ -31,6 +30,7 @@ import ( "github.com/apisix/manager-api/internal/core/entity" "github.com/apisix/manager-api/internal/core/storage" + "github.com/apisix/manager-api/log" ) type Interface interface { @@ -60,12 +60,15 @@ type GenericStoreOption struct { func NewGenericStore(opt GenericStoreOption) (*GenericStore, error) { if opt.BasePath == "" { + log.Warn("base path empty") return nil, fmt.Errorf("base path can not be empty") } if opt.ObjType == nil { + log.Warn("object type is nil") return nil, fmt.Errorf("object type can not be nil") } if opt.KeyFunc == nil { + log.Warn("key func is nil") return nil, fmt.Errorf("key func can not be nil") } @@ -73,6 +76,7 @@ func NewGenericStore(opt GenericStoreOption) (*GenericStore, error) { opt.ObjType = opt.ObjType.Elem() } if opt.ObjType.Kind() != reflect.Struct { + log.Warn("obj type is invalid") return nil, fmt.Errorf("obj type is invalid") } s := &GenericStore{ @@ -106,7 +110,7 @@ func (s *GenericStore) Init() error { go func() { for event := range ch { if event.Canceled { - log.Println("watch failed", event.Error) + log.Warnf("watch failed: %w", event.Error) } for i := range event.Events { @@ -114,7 +118,7 @@ func (s *GenericStore) Init() error { case storage.EventTypePut: objPtr, err := s.StringToObjPtr(event.Events[i].Value) if err != nil { - log.Println("value convert to obj failed", err) + log.Warnf("value convert to obj failed: %w", err) continue } s.cache.Store(event.Events[i].Key[len(s.opt.BasePath)+1:], objPtr) @@ -131,6 +135,7 @@ func (s *GenericStore) Init() error { func (s *GenericStore) Get(key string) (interface{}, error) { ret, ok := s.cache.Load(key) if !ok { + log.Warnf("data not found by key: %s", key) return nil, data.ErrNotFound } return ret, nil @@ -213,6 +218,7 @@ func (s *GenericStore) List(input ListInput) (*ListOutput, error) { func (s *GenericStore) ingestValidate(obj interface{}) (err error) { if s.opt.Validator != nil { if err := s.opt.Validator.Validate(obj); err != nil { + log.Infof("data validate fail: %w", err) return err } } @@ -244,11 +250,13 @@ func (s *GenericStore) Create(ctx context.Context, obj interface{}) error { } _, ok := s.cache.Load(key) if ok { + log.Warnf("key: %s is conflicted", key) return fmt.Errorf("key: %s is conflicted", key) } bs, err := json.Marshal(obj) if err != nil { + log.Warnf("json marshal failed: %s", err) return fmt.Errorf("json marshal failed: %s", err) } if err := s.Stg.Create(ctx, s.GetObjStorageKey(obj), string(bs)); err != nil { @@ -272,6 +280,7 @@ func (s *GenericStore) Update(ctx context.Context, obj interface{}, createIfNotE if createIfNotExist { return s.Create(ctx, obj) } + log.Warnf("key: %s is not found", key) return fmt.Errorf("key: %s is not found", key) } @@ -284,6 +293,7 @@ func (s *GenericStore) Update(ctx context.Context, obj interface{}, createIfNotE bs, err := json.Marshal(obj) if err != nil { + log.Warnf("json marshal failed: %s", err) return fmt.Errorf("json marshal failed: %s", err) } if err := s.Stg.Update(ctx, s.GetObjStorageKey(obj), string(bs)); err != nil { @@ -311,6 +321,7 @@ func (s *GenericStore) StringToObjPtr(str string) (interface{}, error) { objPtr := reflect.New(s.opt.ObjType) err := json.Unmarshal([]byte(str), objPtr.Interface()) if err != nil { + log.Warnf("json marshal failed: %s", err) return nil, fmt.Errorf("json unmarshal failed: %w", err) } diff --git a/api/internal/core/store/storehub.go b/api/internal/core/store/storehub.go index b2685c9ad9..c38286eeb8 100644 --- a/api/internal/core/store/storehub.go +++ b/api/internal/core/store/storehub.go @@ -22,6 +22,7 @@ import ( "github.com/apisix/manager-api/internal/core/entity" "github.com/apisix/manager-api/internal/utils" + "github.com/apisix/manager-api/log" ) type HubKey string @@ -50,9 +51,11 @@ func InitStore(key HubKey, opt GenericStoreOption) error { } s, err := NewGenericStore(opt) if err != nil { + log.Warnf("NewGenericStore error: %w", err) return err } if err := s.Init(); err != nil { + log.Warnf("GenericStore init error: %w", err) return err } diff --git a/api/internal/core/store/validate.go b/api/internal/core/store/validate.go index 14470e04df..b4f3bd1c4e 100644 --- a/api/internal/core/store/validate.go +++ b/api/internal/core/store/validate.go @@ -77,11 +77,13 @@ type APISIXJsonSchemaValidator struct { func NewAPISIXJsonSchemaValidator(jsonPath string) (Validator, error) { schemaDef := conf.Schema.Get(jsonPath).String() if schemaDef == "" { + log.Warnf("scheme validate failed: schema not found, path: %s", jsonPath) return nil, fmt.Errorf("scheme validate failed: schema not found, path: %s", jsonPath) } s, err := gojsonschema.NewSchema(gojsonschema.NewStringLoader(schemaDef)) if err != nil { + log.Warnf("new schema failed: %w", err) return nil, fmt.Errorf("new schema failed: %w", err) } return &APISIXJsonSchemaValidator{ @@ -216,6 +218,7 @@ func checkConf(reqBody interface{}) error { func (v *APISIXJsonSchemaValidator) Validate(obj interface{}) error { ret, err := v.schema.Validate(gojsonschema.NewGoLoader(obj)) if err != nil { + log.Warnf("scheme validate failed: %w", err) return fmt.Errorf("scheme validate failed: %w", err) } @@ -245,12 +248,14 @@ func (v *APISIXJsonSchemaValidator) Validate(obj interface{}) error { schemaDef = conf.Schema.Get("plugins." + pluginName + ".schema").String() } if schemaDef == "" { + log.Warnf("scheme validate failed: schema not found, path: %s", "plugins."+pluginName) return fmt.Errorf("scheme validate failed: schema not found, path: %s", "plugins."+pluginName) } schemaDef = reg.ReplaceAllString(schemaDef, `"properties":{}`) s, err := gojsonschema.NewSchema(gojsonschema.NewStringLoader(schemaDef)) if err != nil { + log.Warnf("init scheme validate failed: %w", err) return fmt.Errorf("scheme validate failed: %w", err) } diff --git a/api/log/zap.go b/api/log/zap.go index 455822ef97..1ae2405a92 100644 --- a/api/log/zap.go +++ b/api/log/zap.go @@ -21,7 +21,7 @@ func init() { logger = zapLogger.Sugar() } -func getLogLevel() zapcore.LevelEnabler{ +func getLogLevel() zapcore.LevelEnabler { level := zapcore.WarnLevel switch conf.ErrorLogLevel { case "debug": @@ -47,9 +47,9 @@ func rotateWriter() zapcore.WriteSyncer { interval := time.Duration(conf.LogRotateInterval) logf, _ := rotatelogs.New( - conf.ErrorLogPath + ".%Y%m%d%H%M%S", - rotatelogs.WithMaxAge(maxAge * time.Second), - rotatelogs.WithRotationTime(interval * time.Second), + conf.ErrorLogPath+".%Y%m%d%H%M%S", + rotatelogs.WithMaxAge(maxAge*time.Second), + rotatelogs.WithRotationTime(interval*time.Second), //rotatelogs.WithRotationCount(7), rotatelogs.WithRotationSize(maxSize), ) @@ -57,7 +57,6 @@ func rotateWriter() zapcore.WriteSyncer { return zapcore.AddSync(logf) } - func getZapFields(logger *zap.SugaredLogger, fields []interface{}) *zap.SugaredLogger { if len(fields) == 0 { return logger From fe5d13d38354b224377c01b41d053a8bb6968d66 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 16:14:44 +0800 Subject: [PATCH 04/21] fix default config --- api/conf/conf.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/conf/conf.json b/api/conf/conf.json index 0cdecc4e68..ee8f7c7029 100644 --- a/api/conf/conf.json +++ b/api/conf/conf.json @@ -13,12 +13,12 @@ }, "log": { "error_log": { - "level": "info", + "level": "warn", "file_path": "./logs/error.log" }, "rotate": { - "interval": 20, - "max_size": 100, + "interval": 86400, + "max_size": 100000000, "max_age": 2592000 } } From 550eda4e9e0939a2d00f3c9b693d5e3ea29c0046 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 17:16:13 +0800 Subject: [PATCH 05/21] fix CI fail --- api/filter/logging.go | 68 +------------------------------------------ api/log/zap.go | 43 +++++++++++++++------------ 2 files changed, 26 insertions(+), 85 deletions(-) diff --git a/api/filter/logging.go b/api/filter/logging.go index de4acda2de..e8ada32025 100644 --- a/api/filter/logging.go +++ b/api/filter/logging.go @@ -16,70 +16,4 @@ */ package filter -import ( - "bytes" - - "github.com/gin-gonic/gin" -) - -//func RequestLogHandler() gin.HandlerFunc { -// //return func(c *gin.Context) { -// // start, host, remoteIP, path, method := time.Now(), c.Request.Host, c.ClientIP(), c.Request.URL.Path, c.Request.Method -// // var val interface{} -// // if method == "GET" { -// // val = c.Request.URL.Query() -// // } else { -// // val, _ = c.GetRawData() -// // -// // // set RequestBody back -// // c.Request.Body = ioutil.NopCloser(bytes.NewReader(val.([]byte))) -// // } -// // c.Set("requestBody", val) -// // uuid, _ := c.Get("X-Request-Id") -// // -// // param, _ := c.Get("requestBody") -// // -// // switch paramType := param.(type) { -// // case []byte: -// // param = string(param.([]byte)) -// // log.Infof("type of param: %#v", paramType) -// // default: -// // } -// // -// // blw := &bodyLogWriter{body: bytes.NewBufferString(""), ResponseWriter: c.Writer} -// // c.Writer = blw -// // c.Next() -// // latency := time.Since(start) / 1000000 -// // statusCode := c.Writer.Status() -// // respBody := blw.body.String() -// // if uuid == "" { -// // uuid = c.Writer.Header().Get("X-Request-Id") -// // } -// // var errs []string -// // for _, err := range c.Errors { -// // errs = append(errs, err.Error()) -// // } -// // logger.With(logrus.Fields{ -// // "requestId": uuid, -// // "latency": latency, -// // "remoteIp": remoteIP, -// // "method": method, -// // "path": path, -// // "statusCode": statusCode, -// // "host": host, -// // "params": param, -// // "respBody": respBody, -// // "errMsg": errs, -// // }).Info("") -// //} -//} - -type bodyLogWriter struct { - gin.ResponseWriter - body *bytes.Buffer -} - -func (w bodyLogWriter) Write(b []byte) (int, error) { - w.body.Write(b) - return w.ResponseWriter.Write(b) -} +//for logging access log, will refactor it in a new pr. \ No newline at end of file diff --git a/api/log/zap.go b/api/log/zap.go index 1ae2405a92..3de9007741 100644 --- a/api/log/zap.go +++ b/api/log/zap.go @@ -1,17 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package log import ( - "github.com/apisix/manager-api/conf" - rotatelogs "github.com/lestrrat-go/file-rotatelogs" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "time" + + "github.com/apisix/manager-api/conf" ) var logger *zap.SugaredLogger func init() { - writeSyncer := rotateWriter() + writeSyncer := fileWriter() encoder := getEncoder() logLevel := getLogLevel() core := zapcore.NewCore(encoder, writeSyncer, logLevel) @@ -41,20 +56,12 @@ func getEncoder() zapcore.Encoder { return zapcore.NewConsoleEncoder(encoderConfig) } -func rotateWriter() zapcore.WriteSyncer { - maxAge := time.Duration(conf.LogRotateMaxAge) - maxSize := conf.LogRotateMaxSize - interval := time.Duration(conf.LogRotateInterval) - - logf, _ := rotatelogs.New( - conf.ErrorLogPath+".%Y%m%d%H%M%S", - rotatelogs.WithMaxAge(maxAge*time.Second), - rotatelogs.WithRotationTime(interval*time.Second), - //rotatelogs.WithRotationCount(7), - rotatelogs.WithRotationSize(maxSize), - ) - - return zapcore.AddSync(logf) +func fileWriter() zapcore.WriteSyncer { + writer, _, err := zap.Open(conf.ErrorLogPath) + if err != nil { + panic(err) + } + return writer } func getZapFields(logger *zap.SugaredLogger, fields []interface{}) *zap.SugaredLogger { From 9b97ea15204a03474a6c47842e1c6ca3bec62801 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 17:41:37 +0800 Subject: [PATCH 06/21] fix: should not save log to file by default --- api/conf/conf.go | 2 +- api/conf/conf.json | 4 ++-- api/filter/logging.go | 2 +- api/log/zap.go | 7 +++++++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/api/conf/conf.go b/api/conf/conf.go index d41bf7a56a..4336409b8a 100644 --- a/api/conf/conf.go +++ b/api/conf/conf.go @@ -49,7 +49,7 @@ var ( ServerPort = 80 ETCDEndpoints = "127.0.0.1:2379" ErrorLogLevel = "warn" - ErrorLogPath = "./logs/error.log" + ErrorLogPath = "" LogRotateInterval int64 = 86400 //second LogRotateMaxSize int64 = 100000000 //byte LogRotateMaxAge int64 = 2592000 //second diff --git a/api/conf/conf.json b/api/conf/conf.json index ee8f7c7029..e478663b02 100644 --- a/api/conf/conf.json +++ b/api/conf/conf.json @@ -13,8 +13,8 @@ }, "log": { "error_log": { - "level": "warn", - "file_path": "./logs/error.log" + "level": "info", + "file_path": "" }, "rotate": { "interval": 86400, diff --git a/api/filter/logging.go b/api/filter/logging.go index e8ada32025..328ac4dc6d 100644 --- a/api/filter/logging.go +++ b/api/filter/logging.go @@ -16,4 +16,4 @@ */ package filter -//for logging access log, will refactor it in a new pr. \ No newline at end of file +//for logging access log, will refactor it in a new pr. diff --git a/api/log/zap.go b/api/log/zap.go index 3de9007741..a77c4bf39f 100644 --- a/api/log/zap.go +++ b/api/log/zap.go @@ -17,6 +17,8 @@ package log import ( + "os" + "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -57,6 +59,11 @@ func getEncoder() zapcore.Encoder { } func fileWriter() zapcore.WriteSyncer { + //if not config file path + if conf.ErrorLogPath == "" { + return zapcore.Lock(os.Stdout) + } + writer, _, err := zap.Open(conf.ErrorLogPath) if err != nil { panic(err) From 654fabbe977af6324acc36aabace4bcedcbb5c13 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 19:13:23 +0800 Subject: [PATCH 07/21] test: add test case --- .github/workflows/cli_test.yml | 29 +++++++++++++++++++ api/conf/conf.json | 2 +- api/main.go | 2 +- api/test/shell/cli_test.sh | 51 ++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/cli_test.yml create mode 100755 api/test/shell/cli_test.sh diff --git a/.github/workflows/cli_test.yml b/.github/workflows/cli_test.yml new file mode 100644 index 0000000000..d5533117c7 --- /dev/null +++ b/.github/workflows/cli_test.yml @@ -0,0 +1,29 @@ +name: Test API + +on: + push: + branches: + - master + pull_request: + branches: + - master + - v2.0 + +jobs: + run-test: + runs-on: ubuntu-latest + + services: + etcd: + image: bitnami/etcd:3.4.13 + ports: + - 2379:2379 + - 2380:2380 + env: + ALLOW_NONE_AUTHENTICATION: yes + + steps: + - uses: actions/checkout@v2 + + - name: run test + run: sudo ./test/shell/cli_test.sh \ No newline at end of file diff --git a/api/conf/conf.json b/api/conf/conf.json index e478663b02..915aee8746 100644 --- a/api/conf/conf.json +++ b/api/conf/conf.json @@ -13,7 +13,7 @@ }, "log": { "error_log": { - "level": "info", + "level": "warn", "file_path": "" }, "rotate": { diff --git a/api/main.go b/api/main.go index b12c77425f..52f7e64288 100644 --- a/api/main.go +++ b/api/main.go @@ -52,7 +52,7 @@ func main() { log.Infof("The Manager API is listening on %s ", addr) if err := s.ListenAndServe(); err != nil { - log.Error("listen and serv fail: %w", err) + log.Errorf("listen and serv fail: %w", err) } utils.CloseAll() diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh new file mode 100755 index 0000000000..a5e48b319e --- /dev/null +++ b/api/test/shell/cli_test.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# 'make init' operates scripts and related configuration files in the current directory +# The 'apisix' command is a command in the /usr/local/apisix, +# and the configuration file for the operation is in the /usr/local/apisix/conf + +set -ex + +clean_up() { + git checkout conf/conf.json +} + +trap clean_up EXIT + +sed -i 's/"file_path": ""/"file_path": "./error.log"/' conf/conf.json +sed -i 's/warn/info/' conf/conf.json + +go run main.go & + +sleep 3 + +ps -aux | grep "main.go" | awk {'print $2'} | xargs kill -9 + +if [ ! -f "./error.log" ]; then + echo "failed: failed to write log" + exit 1 +fi + +if [ ! cat "./error.log" | grep INFO ]; then + echo "failed: failed to write log on right level" + exit 1 +fi + + From 61a186bb26385e23739d9fd21496f5d7f57cab4f Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 19:46:02 +0800 Subject: [PATCH 08/21] test: add test case --- .github/workflows/cli_test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli_test.yml b/.github/workflows/cli_test.yml index d5533117c7..6cf4f85a47 100644 --- a/.github/workflows/cli_test.yml +++ b/.github/workflows/cli_test.yml @@ -1,9 +1,10 @@ -name: Test API +name: Test CLI on: push: branches: - master + - v2.0 pull_request: branches: - master @@ -26,4 +27,4 @@ jobs: - uses: actions/checkout@v2 - name: run test - run: sudo ./test/shell/cli_test.sh \ No newline at end of file + run: sudo ./api/test/shell/cli_test.sh \ No newline at end of file From 6c07af547263f30815dfca79513aab3a070b387d Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 19:48:45 +0800 Subject: [PATCH 09/21] fix CI fail --- .github/workflows/cli_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli_test.yml b/.github/workflows/cli_test.yml index 6cf4f85a47..01c56e402f 100644 --- a/.github/workflows/cli_test.yml +++ b/.github/workflows/cli_test.yml @@ -27,4 +27,5 @@ jobs: - uses: actions/checkout@v2 - name: run test - run: sudo ./api/test/shell/cli_test.sh \ No newline at end of file + working-directory: ./api + run: sudo ./test/shell/cli_test.sh From f64444ad8eb7d55062bb3f507baeb820e9ec432e Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 20:00:58 +0800 Subject: [PATCH 10/21] fix error --- api/test/shell/cli_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh index a5e48b319e..7689755e20 100755 --- a/api/test/shell/cli_test.sh +++ b/api/test/shell/cli_test.sh @@ -29,7 +29,7 @@ clean_up() { trap clean_up EXIT -sed -i 's/"file_path": ""/"file_path": "./error.log"/' conf/conf.json +sed -i 's/"file_path": ""/"file_path": ".\/error.log"/' conf/conf.json sed -i 's/warn/info/' conf/conf.json go run main.go & From 01edc75d38b92ff63f607e7871cfd1ecf56e6421 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 20:23:15 +0800 Subject: [PATCH 11/21] fix CI --- api/test/shell/cli_test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh index 7689755e20..59b7605670 100755 --- a/api/test/shell/cli_test.sh +++ b/api/test/shell/cli_test.sh @@ -36,8 +36,6 @@ go run main.go & sleep 3 -ps -aux | grep "main.go" | awk {'print $2'} | xargs kill -9 - if [ ! -f "./error.log" ]; then echo "failed: failed to write log" exit 1 From 8ce2be605218fb7cc3992e2eb18ccf3e508ac991 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 4 Nov 2020 21:19:50 +0800 Subject: [PATCH 12/21] fix error --- api/test/shell/cli_test.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh index 59b7605670..5f8ba67409 100755 --- a/api/test/shell/cli_test.sh +++ b/api/test/shell/cli_test.sh @@ -32,16 +32,19 @@ trap clean_up EXIT sed -i 's/"file_path": ""/"file_path": ".\/error.log"/' conf/conf.json sed -i 's/warn/info/' conf/conf.json -go run main.go & +export GO111MODULE=on +go build -o ./manager-api . + +./manager-api & sleep 3 -if [ ! -f "./error.log" ]; then +if [[ ! -f "./error.log" ]]; then echo "failed: failed to write log" exit 1 fi -if [ ! cat "./error.log" | grep INFO ]; then +if [[ `grep -c "INFO" ./error.log` -eq '0' ]]; then echo "failed: failed to write log on right level" exit 1 fi From 50a948bcb5e46aa028caf097bb01dd58b88d3246 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Thu, 5 Nov 2020 14:48:20 +0800 Subject: [PATCH 13/21] fix according to reviews --- api/conf/conf.json | 7 +------ api/log/zap.go | 6 ++++++ api/logs/.gitkeep | 0 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 api/logs/.gitkeep diff --git a/api/conf/conf.json b/api/conf/conf.json index 915aee8746..fa3b1993f4 100644 --- a/api/conf/conf.json +++ b/api/conf/conf.json @@ -14,12 +14,7 @@ "log": { "error_log": { "level": "warn", - "file_path": "" - }, - "rotate": { - "interval": 86400, - "max_size": 100000000, - "max_age": 2592000 + "file_path": "logs/error.log" } } }, diff --git a/api/log/zap.go b/api/log/zap.go index a77c4bf39f..47bfd85c37 100644 --- a/api/log/zap.go +++ b/api/log/zap.go @@ -45,8 +45,14 @@ func getLogLevel() zapcore.LevelEnabler { level = zapcore.DebugLevel case "info": level = zapcore.InfoLevel + case "warn": + level = zapcore.WarnLevel case "error": level = zapcore.ErrorLevel + case "panic": + level = zapcore.PanicLevel + case "fatal": + level = zapcore.FatalLevel } return level } diff --git a/api/logs/.gitkeep b/api/logs/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 From e9ca2c2da7d6746b192ac13848ae089abf888b7b Mon Sep 17 00:00:00 2001 From: nic-chen Date: Thu, 5 Nov 2020 15:13:23 +0800 Subject: [PATCH 14/21] test: more test cases --- api/test/shell/cli_test.sh | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh index 5f8ba67409..1ee1dedb81 100755 --- a/api/test/shell/cli_test.sh +++ b/api/test/shell/cli_test.sh @@ -29,15 +29,31 @@ clean_up() { trap clean_up EXIT -sed -i 's/"file_path": ""/"file_path": ".\/error.log"/' conf/conf.json -sed -i 's/warn/info/' conf/conf.json - export GO111MODULE=on go build -o ./manager-api . +#default level: warn, path: logs/error.log ./manager-api & +sleep 3 +pkill -f manager-api +if [[ ! -f "./logs/error.log" ]]; then + echo "failed: failed to write log" + exit 1 +fi + +if [[ `grep -c "INFO" ./error.log` -neq '0' ]]; then + echo "failed: should not write info log when level is warn" + exit 1 +fi + +#change level and path +sed -i 's/"file_path": ""/"file_path": ".\/error.log"/' conf/conf.json +sed -i 's/warn/info/' conf/conf.json + +./manager-api & sleep 3 +pkill -f manager-api if [[ ! -f "./error.log" ]]; then echo "failed: failed to write log" @@ -48,5 +64,3 @@ if [[ `grep -c "INFO" ./error.log` -eq '0' ]]; then echo "failed: failed to write log on right level" exit 1 fi - - From 3382aa39dcc18e205b76ccf591df8a65393a6b4e Mon Sep 17 00:00:00 2001 From: nic-chen Date: Thu, 5 Nov 2020 15:50:02 +0800 Subject: [PATCH 15/21] fix error --- api/test/shell/cli_test.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh index 1ee1dedb81..b079ea04ac 100755 --- a/api/test/shell/cli_test.sh +++ b/api/test/shell/cli_test.sh @@ -17,10 +17,6 @@ # limitations under the License. # -# 'make init' operates scripts and related configuration files in the current directory -# The 'apisix' command is a command in the /usr/local/apisix, -# and the configuration file for the operation is in the /usr/local/apisix/conf - set -ex clean_up() { @@ -33,6 +29,7 @@ export GO111MODULE=on go build -o ./manager-api . #default level: warn, path: logs/error.log + ./manager-api & sleep 3 pkill -f manager-api @@ -42,13 +39,14 @@ if [[ ! -f "./logs/error.log" ]]; then exit 1 fi -if [[ `grep -c "INFO" ./error.log` -neq '0' ]]; then +if [[ `grep -c "INFO" ./error.log` -ne '0' ]]; then echo "failed: should not write info log when level is warn" exit 1 fi #change level and path -sed -i 's/"file_path": ""/"file_path": ".\/error.log"/' conf/conf.json + +sed -i 's/"file_path": "logs\/error.log"/"file_path": ".\/error.log"/' conf/conf.json sed -i 's/warn/info/' conf/conf.json ./manager-api & From acfabc344f066540e9785e94423b30f8675f1caf Mon Sep 17 00:00:00 2001 From: nic-chen Date: Thu, 5 Nov 2020 21:30:23 +0800 Subject: [PATCH 16/21] chore: use `/dev/stdout` as default log file path --- api/conf/conf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/conf/conf.json b/api/conf/conf.json index fa3b1993f4..4a1f825c33 100644 --- a/api/conf/conf.json +++ b/api/conf/conf.json @@ -14,7 +14,7 @@ "log": { "error_log": { "level": "warn", - "file_path": "logs/error.log" + "file_path": "/dev/stdout" } } }, From f3c5dc775aa5b971e27bda6c03ea3b687d488dc3 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Thu, 5 Nov 2020 22:29:44 +0800 Subject: [PATCH 17/21] fix typo --- api/test/shell/cli_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh index b079ea04ac..3c60387fe3 100755 --- a/api/test/shell/cli_test.sh +++ b/api/test/shell/cli_test.sh @@ -39,7 +39,7 @@ if [[ ! -f "./logs/error.log" ]]; then exit 1 fi -if [[ `grep -c "INFO" ./error.log` -ne '0' ]]; then +if [[ `grep -c "INFO" ./logs/error.log` -ne '0' ]]; then echo "failed: should not write info log when level is warn" exit 1 fi From ce70229217259584af67763653e119299ac83474 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Fri, 6 Nov 2020 09:35:04 +0800 Subject: [PATCH 18/21] fix docker for logs dir --- Dockerfile | 2 ++ api/Dockerfile | 2 ++ api/logs/{.gitkeep => placeholder.txt} | 0 3 files changed, 4 insertions(+) rename api/logs/{.gitkeep => placeholder.txt} (100%) diff --git a/Dockerfile b/Dockerfile index 2baad1cf73..a1e219f4da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,6 +77,8 @@ COPY --from=api-builder /usr/local/apisix-dashboard/output/ ./ COPY --from=fe-builder /usr/local/apisix-dashboard/output/ ./ +RUN mkdir logs + EXPOSE 8080 CMD [ "/usr/local/apisix-dashboard/manager-api" ] diff --git a/api/Dockerfile b/api/Dockerfile index 426e7c6dee..eb56b8e862 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -54,6 +54,8 @@ WORKDIR /go/manager-api COPY --from=build-env /go/manager-api/ /go/manager-api/ COPY --from=build-env /usr/share/zoneinfo/Hongkong /etc/localtime +RUN mkdir logs + EXPOSE 8080 RUN chmod +x ./entry.sh diff --git a/api/logs/.gitkeep b/api/logs/placeholder.txt similarity index 100% rename from api/logs/.gitkeep rename to api/logs/placeholder.txt From d0d72fed01927278e48d53a8fd4e6484056cf1c2 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Fri, 6 Nov 2020 09:45:06 +0800 Subject: [PATCH 19/21] fix CI fail --- api/test/shell/cli_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh index 3c60387fe3..5be0146213 100755 --- a/api/test/shell/cli_test.sh +++ b/api/test/shell/cli_test.sh @@ -46,8 +46,8 @@ fi #change level and path -sed -i 's/"file_path": "logs\/error.log"/"file_path": ".\/error.log"/' conf/conf.json -sed -i 's/warn/info/' conf/conf.json +sed -i 's/file_path: logs\/error.log/file_path: .\/error.log/' conf/conf.yaml +sed -i 's/warn/info/' conf/conf.yaml ./manager-api & sleep 3 From 2e2ac01847b3ab2f34623b1c3279a31641f7d4c8 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Fri, 6 Nov 2020 09:51:22 +0800 Subject: [PATCH 20/21] fix: delete useless files --- api/conf/conf.json | 37 ------------------------------------- api/conf/conf_preview.json | 23 ----------------------- 2 files changed, 60 deletions(-) delete mode 100644 api/conf/conf.json delete mode 100644 api/conf/conf_preview.json diff --git a/api/conf/conf.json b/api/conf/conf.json deleted file mode 100644 index 4a1f825c33..0000000000 --- a/api/conf/conf.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "conf": { - "syslog": { - "host": "127.0.0.1" - }, - "listen": { - "host": "127.0.0.1", - "port": 8080 - }, - "dag-lib-path": "", - "etcd": { - "endpoints": "127.0.0.1:2379" - }, - "log": { - "error_log": { - "level": "warn", - "file_path": "/dev/stdout" - } - } - }, - "authentication": { - "session": { - "secret": "secret", - "expireTime": 3600 - }, - "user": [ - { - "username": "admin", - "password": "admin" - }, - { - "username": "user", - "password": "user" - } - ] - } -} diff --git a/api/conf/conf_preview.json b/api/conf/conf_preview.json deleted file mode 100644 index eef8683d2a..0000000000 --- a/api/conf/conf_preview.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "conf": { - "syslog": { - "host": "#syslogAddress#" - } - }, - "authentication": { - "session": { - "secret": "secret", - "expireTime": 3600 - }, - "user": [ - { - "username": "admin", - "password": "admin" - }, - { - "username": "user", - "password": "user" - } - ] - } -} From 30d8b7370adf9faead8260a5bdea2472190c1a15 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Fri, 6 Nov 2020 09:53:58 +0800 Subject: [PATCH 21/21] fix: change file name --- api/test/shell/cli_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh index 5be0146213..5023df3f94 100755 --- a/api/test/shell/cli_test.sh +++ b/api/test/shell/cli_test.sh @@ -20,7 +20,7 @@ set -ex clean_up() { - git checkout conf/conf.json + git checkout conf/conf.yaml } trap clean_up EXIT