Merge branch 'master' into la--add-create-pv-permissions

cmd/http-service/Makefile

@@ -0,0 +1,23 @@
+LDFLAGS += -X "github.com/pingcap/tidb-operator/http-service/version.ReleaseVersion=$(shell git describe --tags --dirty="-dev")"
+LDFLAGS += -X "github.com/pingcap/tidb-operator/http-service/version.BuildTS=$(shell date -u '+%Y-%m-%d %H:%M:%S')"
+LDFLAGS += -X "github.com/pingcap/tidb-operator/http-service/version.GitHash=$(shell git rev-parse HEAD)"
+LDFLAGS += -X "github.com/pingcap/tidb-operator/http-service/version.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)"
+LDFLAGS += -X "github.com/pingcap/tidb-operator/http-service/version.GoVersion=$(shell go version)"
+
+GO       := $(GOENV) go
+GOBUILD  := CGO_ENABLED=0 $(GO) build
+
+build:
+	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/http-service .
+
+buf:
+	GOBIN=$(shell pwd)/bin/ $(GO) install github.com/bufbuild/buf/cmd/buf@v1.26.1
+
+buf-format: buf
+	$(shell pwd)/bin/buf format ./idl/api/service.proto -w
+
+buf-lint: buf buf-format
+	$(shell pwd)/bin/buf lint ./idl
+
+buf-generate: buf buf-format
+	$(shell pwd)/bin/buf generate ./idl

cmd/http-service/README.md

@@ -0,0 +1,28 @@
+# HTTP Service
+
+An HTTP Service to convert Kubernetes APIs into HTTP APIs, so that some users can use HTTP clients to manage TiDB clusters and backups & restores directly.
+
+## Development
+
+### OpenAPI
+
+An OpenAPI Swagger file is generated in the `pbgen/oas` directory, you can view this file via some Swagger tools or upload the file to [Redoc](https://redocly.github.io/redoc/) and then view in the web UI directly.
+
+### Generate gRPC stubs and OpenAPI spec
+
+run `make buf-generate`
+
+### Build
+
+run `make build`
+
+## Run
+
+- When running the binary out of a Kubernetes cluster, the `--kubeconfig` must be set to a KUBECONFIG file path.
+  - The context name in the KUBECONFIG should be used as the `kubernetes-id` in the HTTP header.
+- If the Kubernetes cluster does not have enough resources, set `LOCAL_RUN=true` environment variable when running the binary.
+  - This will let this HTTP Service to remove the CPU & memory requests for components so that Pods can be scheduled.
+
+## Test
+
+There are some JSON files in the `examples` directory which can be used as the HTTP body when testing.

cmd/http-service/buf.gen.yaml

@@ -0,0 +1,18 @@
+version: v1
+plugins:
+  - plugin: buf.build/grpc/go:v1.3.0
+    out: ./pbgen/
+    opt:
+      - paths=source_relative
+  # dependencies
+  - plugin: buf.build/protocolbuffers/go
+    out: ./pbgen/
+    opt:
+      - paths=source_relative
+  - plugin: buf.build/grpc-ecosystem/gateway:v2.18.0
+    out: ./pbgen/
+    opt:
+      - paths=source_relative
+  - plugin: buf.build/grpc-ecosystem/openapiv2:v2.18.0
+    out: ./pbgen/oas/
+    opt: allow_merge=true,merge_file_name=openapi-spec,json_names_for_fields=false,omit_enum_default_value=true

cmd/http-service/config.go

@@ -0,0 +1,79 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/pingcap/tidb-operator/http-service/version"
+)
+
+const (
+	defaultAddr             = ":9080"
+	defaultInternalGRPCAddr = "127.0.0.1:9081"
+	defaultLogLevel         = "info"
+)
+
+type Config struct {
+	flagSet *flag.FlagSet
+
+	// printVersion is a flag to print version information.
+	printVersion bool
+
+	// LogFile is the path of the log file.
+	LogFile string `toml:"log-file" json:"log-file"`
+	// LogLevel is the log level.
+	LogLevel string `toml:"log-level" json:"log-level"`
+
+	// Addr is the address to listen on.
+	Addr string `toml:"addr" json:"addr"`
+
+	// InternalGRPCAddr is the address of internal grpc server.
+	InternalGRPCAddr string `toml:"internal-grpc-addr" json:"internal-grpc-addr"`
+
+	// Kubeconfig is the path to Kubeconfig.
+	Kubeconfig string `toml:"kubeconfig" json:"kubeconfig"`
+}
+
+func NewConfig() *Config {
+	cfg := &Config{
+		flagSet:          flag.NewFlagSet("http-service", flag.ContinueOnError),
+		LogLevel:         defaultLogLevel,
+		Addr:             defaultAddr,
+		InternalGRPCAddr: defaultInternalGRPCAddr,
+	}
+
+	cfg.flagSet.BoolVar(&cfg.printVersion, "V", false, "print version information and exit")
+	cfg.flagSet.StringVar(&cfg.LogFile, "log-file", "", "log file path")
+	cfg.flagSet.StringVar(&cfg.LogLevel, "L", cfg.LogLevel, "log level")
+	cfg.flagSet.StringVar(&cfg.Addr, "addr", cfg.Addr, "address to listen on")
+	cfg.flagSet.StringVar(&cfg.InternalGRPCAddr, "internal-grpc-addr", cfg.InternalGRPCAddr, "address of internal grpc server")
+	cfg.flagSet.StringVar(&cfg.Kubeconfig, "kubeconfig", cfg.Kubeconfig, "path to kubeconfig")
+
+	return cfg
+}
+
+func (c *Config) Parse(args []string) error {
+	err := c.flagSet.Parse(args)
+	if err != nil {
+		return err
+	}
+
+	if c.printVersion {
+		fmt.Println(version.GetRawInfo())
+		return flag.ErrHelp
+	}
+	return nil
+}

cmd/http-service/examples/create-cluster.json

@@ -0,0 +1,65 @@
+{
+    "cluster_id":"tidb-clsuter-123",
+    "version":"v7.1.0",
+    "user":{
+       "username":"root",
+       "password":"123456"
+    },
+    "pd":{
+       "replicas":1,
+       "resource":{
+          "cpu":1,
+          "memory":1,
+          "storage":1
+       },
+       "config":"string",
+       "port":0
+    },
+    "tikv":{
+       "replicas":1,
+       "resource":{
+          "cpu":2,
+          "memory":4,
+          "storage":10
+       },
+       "config":"string",
+       "port":0
+    },
+    "tiflash":{
+       "replicas":1,
+       "resource":{
+          "cpu":2,
+          "memory":4,
+          "storage":5
+       },
+       "config":"string",
+       "port":0
+    },
+    "tidb":{
+       "replicas":1,
+       "resource":{
+          "cpu":2,
+          "memory":4,
+          "storage":0
+       },
+       "config":"string",
+       "port":4000
+    },
+    "prometheus": {
+       "version": "v2.27.1",
+       "resource": {
+          "cpu": 2,
+          "memory": 4
+       },
+       "config": "string"
+    },
+    "grafana": {
+       "version": "7.5.11",
+       "resource": {
+          "cpu": 2,
+          "memory": 4
+       },
+       "envs": {},
+       "port": 3000
+    }
+}

cmd/http-service/go.mod

@@ -0,0 +1,95 @@
+module github.com/pingcap/tidb-operator/http-service
+
+go 1.19
+
+require (
+	github.com/gin-gonic/gin v1.9.1
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0
+	github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22
+	github.com/pingcap/tidb-operator/pkg/apis v1.6.0-alpha.6
+	github.com/pingcap/tidb-operator/pkg/client v1.6.0-alpha.6
+	go.uber.org/zap v1.23.0
+	google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d
+	google.golang.org/grpc v1.58.1
+	google.golang.org/protobuf v1.31.0
+	k8s.io/api v0.23.17
+	k8s.io/apimachinery v0.23.17
+	k8s.io/client-go v0.23.17
+	k8s.io/utils v0.0.0-20211116205334-6203023598ed
+)
+
+replace (
+	github.com/pingcap/tidb-operator/pkg/apis => ../../pkg/apis
+	github.com/pingcap/tidb-operator/pkg/client => ../../pkg/client
+)
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/aws/aws-sdk-go v1.44.72 // indirect
+	github.com/benbjohnson/clock v1.1.0 // indirect
+	github.com/bytedance/sonic v1.10.1 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
+	github.com/chenzhuoyu/iasm v0.9.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful v2.16.0+incompatible // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-logr/logr v1.2.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.15.4 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pingcap/TiProxy/lib v0.0.0-20230201020701-df06ec482c69 // indirect
+	github.com/pingcap/errors v0.11.4 // indirect
+	github.com/prometheus/common v0.28.0 // indirect
+	github.com/prometheus/prometheus v1.8.2 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/arch v0.5.0 // indirect
+	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.23.17 // indirect
+	k8s.io/klog/v2 v2.30.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
+)