*: adapt needs for operator support (#76)

.github/workflows/build.yml

@@ -25,7 +25,7 @@ jobs:
       - name: "setup golang"
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version-file: go.mod
       - name: "try to use build cache"
         uses: actions/cache@v3
         with:

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
       - name: "setup golang"
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version-file: go.mod
       - name: "try to use build cache"
         uses: actions/cache@v3
         with:

.gitignore

@@ -1,4 +1,5 @@
 bin
+test
 *.iml
 .idea
 *.swp
@@ -7,4 +8,3 @@ vendor
 work
 .vscode/
 .coverage*
-.git

Makefile

@@ -15,11 +15,16 @@
 
 PROJECTNAME = $(shell basename "$(PWD)")
 GOBIN := $(shell pwd)/bin
+DOCKERFLAG ?=
+RELEASE ?=
+ifneq ($(RELEASE), "")
+	DOCKERFLAG ?= --squash
+endif
 BUILD_TAGS ?=
 LDFLAGS ?= 
-GOFLAGS ?= -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -tags '${BUILD_TAGS}'
+BUILDFLAGS := $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -tags '${BUILD_TAGS}'
 ifeq ("$(WITH_RACE)", "1")
-	GOFLAGS = $(GOFLAGS) -race
+	BUILDFLAGS = $(BUILDFLAGS) -race
 endif
 IMAGE_TAG ?= latest
 EXECUTABLE_TARGETS := $(patsubst cmd/%,cmd_%,$(wildcard cmd/*))
@@ -30,13 +35,13 @@ default: cmd
 
 cmd: $(EXECUTABLE_TARGETS)
 
-build: cmd
-	go build $(GOFLAGS) ./...
+build:
+	go build $(BUILDFLAGS) ./...
 
 cmd_%: OUTPUT=$(patsubst cmd_%,./bin/%,$@)
 cmd_%: SOURCE=$(patsubst cmd_%,./cmd/%,$@)
 cmd_%:
-	go build $(GOFLAGS) -o $(OUTPUT) $(SOURCE)
+	go build $(BUILDFLAGS) -o $(OUTPUT) $(SOURCE)
 
 test:
 	go test -coverprofile=.coverage.out ./...
@@ -45,4 +50,4 @@ test:
 	go tool cover -html=.coverage.out -o .coverage.html
 
 docker:
-	docker build -t "tiproxy:${IMAGE_TAG}" -f docker/Dockerfile .
+	docker build $(DOCKERFLAG) -t "tiproxy:${IMAGE_TAG}" --build-arg='GOPROXY=$(shell go env GOPROXY),BUILDFLAGS=$(BUILDFLAGS),' -f docker/Dockerfile .

cmd/weirctl/main.go

@@ -15,42 +15,11 @@
 package main
 
 import (
-	"net/http"
-
-	"github.com/pingcap/TiProxy/pkg/util/cmd"
-	"github.com/spf13/cobra"
-	"go.uber.org/zap"
+	"github.com/pingcap/TiProxy/lib/cli"
+	"github.com/pingcap/TiProxy/lib/util/cmd"
 )
 
 func main() {
-	rootCmd := &cobra.Command{
-		Use:   "weirctl",
-		Short: "cli",
-	}
-
-	ctx := &Context{}
-
-	curls := rootCmd.PersistentFlags().StringArray("curls", []string{"localhost:3080"}, "API gateway addresses")
-	logEncoder := rootCmd.PersistentFlags().String("log_encoder", "tidb", "log in format of tidb, console, or json")
-	logLevel := rootCmd.PersistentFlags().String("log_level", "info", "log level")
-	rootCmd.PersistentFlags().Bool("indent", true, "whether indent the returned json")
-	rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
-		zapcfg := zap.NewDevelopmentConfig()
-		zapcfg.Encoding = *logEncoder
-		if level, err := zap.ParseAtomicLevel(*logLevel); err == nil {
-			zapcfg.Level = level
-		}
-		logger, err := zapcfg.Build()
-		if err != nil {
-			return err
-		}
-		ctx.Logger = logger.Named("cli")
-		ctx.Client = &http.Client{}
-		ctx.CUrls = *curls
-		return nil
-	}
-
-	rootCmd.AddCommand(GetNamespaceCmd(ctx))
-	rootCmd.AddCommand(GetConfigCmd(ctx))
+	rootCmd := cli.GetRootCmd()
 	cmd.RunRootCommand(rootCmd)
 }

cmd/weirproxy/main.go

@@ -18,10 +18,11 @@ package main
 import (
 	"io/ioutil"
 
-	"github.com/pingcap/TiProxy/pkg/config"
+	"github.com/pingcap/TiProxy/lib/config"
 	"github.com/pingcap/TiProxy/pkg/server"
-	"github.com/pingcap/TiProxy/pkg/util/cmd"
-	"github.com/pingcap/TiProxy/pkg/util/waitgroup"
+	"github.com/pingcap/TiProxy/lib/util/cmd"
+	"github.com/pingcap/TiProxy/lib/util/errors"
+	"github.com/pingcap/TiProxy/lib/util/waitgroup"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 )
@@ -33,6 +34,7 @@ func main() {
 	}
 
 	configFile := rootCmd.PersistentFlags().String("config", "conf/weirproxy.yaml", "weir proxy config file path")
+	pubAddr := rootCmd.PersistentFlags().String("pub_addr", "127.0.0.1", "IP or domain, will be used as the accessible addr for other clients")
 	logEncoder := rootCmd.PersistentFlags().String("log_encoder", "", "log in format of tidb, console, or json")
 	logLevel := rootCmd.PersistentFlags().String("log_level", "", "log level")
 
@@ -66,28 +68,23 @@ func main() {
 		}
 		logger = logger.Named("main")
 
-		srv, err := server.NewServer(cmd.Context(), cfg, logger)
+		srv, err := server.NewServer(cmd.Context(), cfg, logger, *pubAddr)
 		if err != nil {
-			logger.Error("fail to create server", zap.Error(err))
-			return err
+			return errors.Wrapf(err, "fail to create server")
 		}
 
 		var wg waitgroup.WaitGroup
-		wg.Run(func() {
-			if err := srv.Run(cmd.Context()); err != nil {
-				logger.Error("shutdown with error", zap.Error(err))
-			}
-		})
+		wg.Run(func() { srv.Run(cmd.Context()) })
 
 		<-cmd.Context().Done()
 		if e := srv.Close(); e != nil {
-			logger.Error("shutdown with errors", zap.Error(e))
+			err = errors.Wrapf(err, "shutdown with errors")
 		} else {
 			logger.Info("gracefully shutdown")
 		}
 
 		wg.Wait()
-		return nil
+		return err
 	}
 
 	cmd.RunRootCommand(rootCmd)

conf/weirproxy.yaml

@@ -1,22 +1,12 @@
 workdir: "./work"
-listen-urls:
-  - "http://0.0.0.0:3080"
-advertise-urls:
-  - "http://127.0.0.1:3080"
-listen-peer-urls:
-  - "http://0.0.0.0:3081"
-advertise-peer-urls:
-  - "http://127.0.0.1:3081"
-config:
-  ignore-wrong-namespace: true
 proxy:
   addr: "0.0.0.0:6000"
   tcp-keep-alive: true
   max-connections: 1000
   pd-addrs: "127.0.0.1:2379"
 metrics:
-  prom-cluster: "default"
 api:
+  addr: "0.0.0.0:3080"
   enable-basic-auth: false
   user: ""
   password: ""
@@ -30,3 +20,7 @@ log:
     max-backups: 1
 security:
   rsa-key-size: 4096
+advance:
+  # ignore-wrong-namespace: true
+  # peer-port: "3081"
+  # watch-interval: "10m"

docker/Dockerfile

@@ -4,8 +4,15 @@ EXPOSE 3080
 EXPOSE 3081
 EXPOSE 6000
 
-ADD ./ proxy
-RUN sh ./proxy/docker/apk-fastest-mirror.sh
-RUN apk add git make go
-RUN cd proxy && make build && cp bin/* /bin/ && cp -a conf /etc/proxy && cd .. && rm -rf proxy
-RUN apk del git make go
+ADD cmd proxy/cmd
+ADD lib proxy/lib
+ADD pkg proxy/pkg
+ADD conf proxy/conf
+ADD go.* docker/apk-fastest-mirror.sh Makefile proxy/
+RUN sh ./proxy/apk-fastest-mirror.sh
+RUN apk add --no-cache --progress git make go
+ARG BUILDFLAGS
+ARG GOPROXY
+RUN export BUILDFLAGS=${BUILDFLAGS} && export GOPROXY=${GOPROXY} && cd proxy && ls -al && cat Makefile && make cmd && cp bin/* /bin/ && cp -a conf /etc/proxy && cd .. && rm -rf proxy
+RUN rm -rf $(go env GOMODCACHE GOCACHE) && apk del git make go
+ENTRYPOINT ["/bin/weirproxy", "-conf", "/etc/proxy/weirproxy.yaml"]

docker/apk-fastest-mirror.sh

@@ -36,11 +36,7 @@ for URL in $MIRRORS; do
 
 done
 
-if printf '%b' "$DATA" | sort -n | tail -n +2 > $DST; then
-    echo file $DST created
-fi
-
-BEST=$(head -n1 $DST | cut -d ' ' -f2)
+BEST=$(printf '%b' "$DATA" | sort -n | head -n 1 | cut -d ' ' -f2)
 echo "Best mirror is: $BEST"
 
 sed -i -r 's#^http.+/(.+/main)#'"${BEST%/}"'/\1#' /etc/apk/repositories

go.mod

@@ -1,37 +1,34 @@
 module github.com/pingcap/TiProxy
 
-go 1.18
+go 1.19
 
 require (
 	github.com/gin-contrib/pprof v1.4.0
 	github.com/gin-contrib/zap v0.0.2
 	github.com/gin-gonic/gin v1.8.1
 	github.com/go-mysql-org/go-mysql v1.6.0
-	github.com/goccy/go-yaml v1.9.5
-	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
-	github.com/pingcap/log v1.1.0
-	github.com/pingcap/tidb v1.1.0-beta.0.20220819110652-8b5b724d8a93
-	github.com/pingcap/tidb/parser v0.0.0-20220819110652-8b5b724d8a93
-	github.com/prometheus/client_golang v1.12.2
+	github.com/pingcap/TiProxy/lib v0.0.0-00010101000000-000000000000
+	github.com/pingcap/tidb v1.1.0-beta.0.20220908042057-08b1faf2ad1e
+	github.com/pingcap/tidb/parser v0.0.0-20220908042057-08b1faf2ad1e
+	github.com/prometheus/client_golang v1.13.0
 	github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726
 	github.com/spf13/cobra v1.5.0
 	github.com/stretchr/testify v1.8.0
-	go.etcd.io/etcd/api/v3 v3.5.2
-	go.etcd.io/etcd/client/v3 v3.5.2
-	go.etcd.io/etcd/server/v3 v3.5.2
-	go.uber.org/atomic v1.9.0
-	go.uber.org/zap v1.21.0
-	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
-	google.golang.org/grpc v1.45.0
-	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.1
+	go.etcd.io/etcd/api/v3 v3.5.4
+	go.etcd.io/etcd/client/v3 v3.5.4
+	go.etcd.io/etcd/server/v3 v3.5.4
+	go.uber.org/atomic v1.10.0
+	go.uber.org/zap v1.23.0
+	golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0
+	google.golang.org/grpc v1.49.0
 )
 
 require (
 	github.com/BurntSushi/toml v1.2.0 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cockroachdb/datadriven v1.0.0 // indirect
 	github.com/coocood/freecache v1.2.1 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
@@ -40,7 +37,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/fatih/color v1.13.0 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -63,31 +59,34 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/opentracing/basictracer-go v1.0.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
+	github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 // indirect
+	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 // indirect
-	github.com/pingcap/kvproto v0.0.0-20220804022843-f006036b1277 // indirect
-	github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect
+	github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43 // indirect
+	github.com/pingcap/log v1.1.0 // indirect
+	github.com/pingcap/tipb v0.0.0-20220824081009-0714a57aff1d // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
-	github.com/shirou/gopsutil/v3 v3.22.6 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
+	github.com/shirou/gopsutil/v3 v3.22.7 // indirect
 	github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stathat/consistent v1.0.0 // indirect
-	github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f // indirect
+	github.com/tikv/client-go/v2 v2.0.1-0.20220906094532-f867f498456f // indirect
 	github.com/tikv/pd/client v0.0.0-20220725055910-7187a7ab72db // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
@@ -97,10 +96,10 @@ require (
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
-	go.etcd.io/etcd/client/v2 v2.305.2 // indirect
-	go.etcd.io/etcd/pkg/v3 v3.5.2 // indirect
-	go.etcd.io/etcd/raft/v3 v3.5.2 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
+	go.etcd.io/etcd/client/v2 v2.305.4 // indirect
+	go.etcd.io/etcd/pkg/v3 v3.5.4 // indirect
+	go.etcd.io/etcd/raft/v3 v3.5.4 // indirect
 	go.opentelemetry.io/contrib v0.20.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect
 	go.opentelemetry.io/otel v0.20.0 // indirect
@@ -111,16 +110,21 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
 	go.opentelemetry.io/otel/trace v0.20.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.7.0 // indirect
+	go.uber.org/goleak v1.2.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/net v0.0.0-20220907135653-1e95f45603a7 // indirect
+	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+	golang.org/x/sys v0.0.0-20220907062415-87db552b00fd // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect
-	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
-	google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
+	google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )
+
+replace github.com/pingcap/TiProxy/lib => ./lib