chore: refactor conf of manager api

.github/workflows/e2e-test-backend.yml

@@ -16,13 +16,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: install runtime
-        run: |
-          sudo apt-get update
-          sudo add-apt-repository ppa:longsleep/golang-backports
-          sudo apt update
-          export GO111MOUDULE=on
-          sudo apt install golang-1.14-go
+      - name: setup go
+        uses: actions/setup-go@v1
+        with:
+          go-version: '1.13'
 
       - name: run docker compose
         working-directory: ./api/test/docker

.github/workflows/test-api.yml

@@ -28,18 +28,19 @@ jobs:
       - name: get lua lib
         run: |
           wget https://github.com/api7/dag-to-lua/archive/v1.1.tar.gz
-          sudo mkdir -p /go/manager-api/dag-to-lua/
+          sudo mkdir -p ./api/dag-to-lua
           tar -zxvf v1.1.tar.gz
-          sudo mv ./dag-to-lua-1.1/lib/* /go/manager-api/dag-to-lua/
+          sudo mv ./dag-to-lua-1.1/lib/* ./api/dag-to-lua/
 
-      - name: install runtime
-        run: |
-          sudo apt-get update
-          sudo apt-get install lua5.1
-          sudo add-apt-repository ppa:longsleep/golang-backports
-          sudo apt update
-          export GO111MOUDULE=on
-          sudo apt install golang-1.14-go
+      - name: setup go
+        uses: actions/setup-go@v1
+        with:
+          go-version: '1.13'
+
+      - name: setup lua
+        uses: leafo/gh-actions-lua@v7
+        with:
+          luaVersion: "5.1.5"
 
       - name: run test
         run: |
@@ -55,12 +56,12 @@ jobs:
         run: |
           export GO111MOUDULE=on
           export APISIX_CONF_PATH=$PWD/conf
-          sed -i 's/8080/8088/' conf/conf.json
+          sed -i 's/8080/8088/' conf/conf.yaml
           go build -o ./manager-api
           ./manager-api > ./api.log 2>&1 &
           sleep 2
           cat ./api.log
-          cat conf/conf.json
+          cat conf/conf.yaml
 
       - name: run with custom port
         working-directory: ./api

Dockerfile

@@ -75,11 +75,7 @@ WORKDIR /usr/local/apisix-dashboard
 
 COPY --from=api-builder /usr/local/apisix-dashboard/output/ ./
 
-# NOTE: Just to compatible with /api/conf/conf.go WebDir
-RUN mkdir -p ./output
-COPY --from=fe-builder /usr/local/apisix-dashboard/output/ ./output
-
-ENV APISIX_CONF_PATH /usr/local/apisix-dashboard/conf
+COPY --from=fe-builder /usr/local/apisix-dashboard/output/ ./
 
 EXPOSE 8080
 

Makefile

@@ -49,5 +49,5 @@ golang-lint: ## Run the golangci-lint application (install if not found)
 ### api-test:         Run the tests of manager-api
 .PHONY: api-test
 api-test:
-	cd api/ && go test -v -race -cover -coverprofile=coverage.txt -covermode=atomic ./...
+	cd api/ && APISIX_API_WORKDIR=$$PWD go test -v -race -cover -coverprofile=coverage.txt -covermode=atomic ./...
 

api/Dockerfile

@@ -19,15 +19,15 @@ FROM golang:1.13.8 AS build-env
 
 WORKDIR /go/src/github.com/apisix/manager-api
 COPY . .
-RUN mkdir -p /go/manager-api \
+RUN mkdir -p /go/manager-api/conf \
     && mkdir -p /go/manager-api/build-tools \
     && go env -w GOPROXY=https://goproxy.io,direct \
     && export GOPROXY=https://goproxy.io \
     && go build -o /go/manager-api/manager-api \
     && mv /go/src/github.com/apisix/manager-api/entry.sh /go/manager-api/ \
     && mv /go/src/github.com/apisix/manager-api/build-tools/* /go/manager-api/build-tools/ \
-    && mv /go/src/github.com/apisix/manager-api/conf/conf_preview.json /go/manager-api/conf.json \
-    && mv /go/src/github.com/apisix/manager-api/conf/schema.json /go/manager-api/schema.json \
+    && mv /go/src/github.com/apisix/manager-api/conf/conf.yaml /go/manager-api/conf/conf.yaml \
+    && mv /go/src/github.com/apisix/manager-api/conf/schema.json /go/manager-api/conf/schema.json \
     && rm -rf /go/src/github.com/apisix/manager-api \
     && rm -rf /etc/localtime \
     && ln -s  /usr/share/zoneinfo/Hongkong /etc/localtime \

api/build.sh

@@ -19,19 +19,19 @@ set -ex
 export ENV=local
 pwd=`pwd`
 
-rm -rf output && mkdir -p output/conf && mkdir -p output/dag-to-lua-1.1
+rm -rf output && mkdir -p output/conf && mkdir -p output/dag-to-lua
 
 # get dag-to-lua lib
 if [[ ! -f "dag-to-lua-1.1/lib/dag-to-lua.lua" ]]; then
     wget https://github.com/api7/dag-to-lua/archive/v1.1.tar.gz -P /tmp
     tar -zxvf /tmp/v1.1.tar.gz -C /tmp
-    cp -r /tmp/dag-to-lua-1.1/lib ./output/dag-to-lua-1.1
+    cp -r /tmp/dag-to-lua-1.1/lib/* ./output/dag-to-lua
 fi
 
 # build
 cd ./api && go build -o ../output/manager-api . && cd ..
 
 cp ./api/conf/schema.json ./output/conf/schema.json
-cp ./api/conf/conf.json ./output/conf/conf.json
+cp ./api/conf/conf.yaml ./output/conf/conf.yaml
 
 echo "Build the Manager API successfully"

api/conf/conf.go

@@ -17,147 +17,134 @@
 package conf
 
 import (
+	"flag"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
-	"path/filepath"
-	"runtime"
 
 	"github.com/tidwall/gjson"
+	"gopkg.in/yaml.v2"
 
 	"github.com/apisix/manager-api/internal/utils"
 )
 
 const (
-	WebDir = "./output/html"
-
 	EnvPROD  = "prod"
 	EnvBETA  = "beta"
 	EnvDEV   = "dev"
 	EnvLOCAL = "local"
 
-	confJsonPath = "/go/manager-api/conf.json"
-	schemaPath   = "/go/manager-api/schema.json"
+	WebDir = "./html"
 )
 
 var (
 	ENV           string
-	basePath      string
 	Schema        gjson.Result
-	DagLibPath    = "/go/manager-api/dag-to-lua/"
+	WorkDir       = "."
 	ServerHost    = "127.0.0.1"
 	ServerPort    = 80
-	ETCDEndpoints = "127.0.0.1:2379"
+	ETCDEndpoints = []string{"127.0.0.1:2379"}
+	UserList      = make(map[string]User, 2)
+	AuthConf      Authentication
 )
 
+type Etcd struct {
+	Endpoints []string
+}
+
+type Listen struct {
+	Host string
+	Port int
+}
+
+type Conf struct {
+	Etcd   Etcd
+	Listen Listen
+}
+
+type User struct {
+	Username string
+	Password string
+}
+
+type Authentication struct {
+	Secret     string
+	ExpireTime int `yaml:"expire_time"`
+	Users      []User
+}
+
+type Config struct {
+	Conf           Conf
+	Authentication Authentication
+}
+
 func init() {
+	//go test
+	if workDir := os.Getenv("APISIX_API_WORKDIR"); workDir != "" {
+		WorkDir = workDir
+	} else {
+		flag.StringVar(&WorkDir, "p", ".", "current work dir")
+		flag.Parse()
+	}
+
+	setConf()
 	setEnvironment()
-	initAuthentication()
 	initSchema()
-	setConf()
 }
 
 func setConf() {
-	filePath := configurationPath()
+	filePath := WorkDir + "/conf/conf.yaml"
 	if configurationContent, err := ioutil.ReadFile(filePath); err != nil {
 		panic(fmt.Sprintf("fail to read configuration: %s", filePath))
 	} else {
-		configuration := gjson.ParseBytes(configurationContent)
+		//configuration := gjson.ParseBytes(configurationContent)
+		config := Config{}
+		err := yaml.Unmarshal(configurationContent, &config)
+		if err != nil {
+			log.Printf("conf: %s, error: %v", configurationContent, err)
+		}
+
 		//listen
-		serverPort := int(configuration.Get("conf.listen.port").Int())
-		if serverPort != 0 {
-			ServerPort = serverPort
+		if config.Conf.Listen.Port != 0 {
+			ServerPort = config.Conf.Listen.Port
 		}
-		serverHost := configuration.Get("conf.listen.host").String()
-		if serverHost != "" {
-			ServerHost = serverHost
+		if config.Conf.Listen.Host != "" {
+			ServerHost = config.Conf.Listen.Host
 		}
 
-		//dag lib path
-		dagLibPath := configuration.Get("conf.dag-lib-path").String()
-		if dagLibPath != "" {
-			DagLibPath = dagLibPath
-		}
 		//etcd
-		eTCDEndpoints := configuration.Get("conf.etcd.endpoints").String()
-		if eTCDEndpoints != "" {
-			ETCDEndpoints = eTCDEndpoints
+		if len(config.Conf.Etcd.Endpoints) > 0 {
+			ETCDEndpoints = config.Conf.Etcd.Endpoints
 		}
 
+		//auth
+		initAuthentication(config.Authentication)
 	}
 }
 
 func setEnvironment() {
-	if env := os.Getenv("ENV"); env == "" {
-		ENV = EnvLOCAL
-	} else {
+	ENV = EnvPROD
+	if env := os.Getenv("ENV"); env != "" {
 		ENV = env
 	}
-
-	_, basePath, _, _ = runtime.Caller(1)
-}
-
-func configurationPath() string {
-	if confPath := os.Getenv("APISIX_CONF_PATH"); confPath != "" {
-		return filepath.Join(confPath, "/conf.json")
-	} else if ENV == EnvLOCAL {
-		return filepath.Join(filepath.Dir(basePath), "conf.json")
-	} else {
-		return confJsonPath
-	}
-}
-
-func getSchemaPath() string {
-	if confPath := os.Getenv("APISIX_CONF_PATH"); confPath != "" {
-		return filepath.Join(confPath, "/schema.json")
-	} else if ENV == EnvLOCAL {
-		return filepath.Join(filepath.Dir(basePath), "schema.json")
-	} else {
-		return schemaPath
-	}
-}
-
-type user struct {
-	Username string
-	Password string
-}
-
-type authenticationConfig struct {
-	Session struct {
-		Secret     string
-		ExpireTime uint64
-	}
 }
 
-var UserList = make(map[string]user, 1)
-
-var AuthenticationConfig authenticationConfig
-
-func initAuthentication() {
-	filePath := configurationPath()
-	configurationContent, err := ioutil.ReadFile(filePath)
-	if err != nil {
-		panic(fmt.Sprintf("fail to read configuration: %s", filePath))
+func initAuthentication(conf Authentication) {
+	AuthConf = conf
+	if AuthConf.Secret == "secret" {
+		AuthConf.Secret = utils.GetFlakeUidStr()
 	}
 
-	configuration := gjson.ParseBytes(configurationContent)
-	userList := configuration.Get("authentication.user").Array()
+	userList := conf.Users
 	// create user list
 	for _, item := range userList {
-		username := item.Map()["username"].String()
-		password := item.Map()["password"].String()
-		UserList[item.Map()["username"].String()] = user{Username: username, Password: password}
+		UserList[item.Username] = item
 	}
-	AuthenticationConfig.Session.Secret = configuration.Get("authentication.session.secret").String()
-	if "secret" == AuthenticationConfig.Session.Secret {
-		AuthenticationConfig.Session.Secret = utils.GetFlakeUidStr()
-	}
-
-	AuthenticationConfig.Session.ExpireTime = configuration.Get("authentication.session.expireTime").Uint()
 }
 
 func initSchema() {
-	filePath := getSchemaPath()
+	filePath := WorkDir + "/conf/schema.json"
 	if schemaContent, err := ioutil.ReadFile(filePath); err != nil {
 		panic(fmt.Sprintf("fail to read configuration: %s", filePath))
 	} else {

api/conf/conf.yaml

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+conf:
+  listen:
+    host: 127.0.0.1          # `manager api` listening ip or host name
+    port: 8080               # `manager api` listening port
+  etcd:
+    endpoints:               # supports defining multiple etcd host addresses for an etcd cluster
+      - 127.0.0.1:2379
+authentication:
+  secret: secret             # secret for jwt token generation.
+                             # *NOTE*: Highly recommended to modify this value to protect `manager api`.
+                             # if it's default value, when `manager api` start , it will generate a random string to replace it.
+  expire_time: 3600          # jwt token expire time, in second
+  users:
+    - username: admin        # username and password for login `manager api`
+      password: admin
+    - username: user
+      password: user

api/filter/authentication.go

@@ -33,7 +33,7 @@ func Authentication() gin.HandlerFunc {
 
 			// verify token
 			token, err := jwt.ParseWithClaims(tokenStr, &jwt.StandardClaims{}, func(token *jwt.Token) (interface{}, error) {
-				return []byte(conf.AuthenticationConfig.Session.Secret), nil
+				return []byte(conf.AuthConf.Secret), nil
 			})
 
 			errResp := gin.H{

api/go.mod

@@ -27,4 +27,5 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	go.etcd.io/etcd v3.3.25+incompatible
 	go.uber.org/zap v1.16.0
+	gopkg.in/yaml.v2 v2.3.0
 )