diff --git a/.gitignore b/.gitignore index 66fd13c9..afda73b8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ # Dependency directories (remove the comment below to include it) # vendor/ + +# MacOS files +.DS_Store +*/.DS_Store \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..89b29d86 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: +- repo: git://github.com/dnephin/pre-commit-golang + rev: v0.4.0 + hooks: + - id: go-fmt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7c96d996 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +##################################### stage 1 ##################################### +FROM bfe + +# adapt BFE conf in docker +COPY output/adapt_bfe_docker.sh /adapt_bfe_docker.sh +RUN sh /adapt_bfe_docker.sh + +# pack /home/work directory for COPY in stage 2 +COPY build/output/bfe_ingress_controller \ + output/ingress.commit \ + /home/work/bfe/bin/ +COPY build/output/start.sh /home/work/start.sh + +##################################### stage 2 ##################################### +# build image from CentOS to reduce image size +FROM centos +RUN yum install -y wget unzip vi net-tools redhat-lsb-core nmap-ncat.x86_64 tcpdump tree jre epel-release \ + && yum install -y supervisor \ + && yum clean all + +# copy base files from bfe image +RUN groupadd -g 501 work && useradd -g work -G work -u 500 -d /home/work work +COPY --from=bfe --chown=work:work /home/work /home/work + +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 +ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/usr/lib/jvm/jre-openjdk/lib/amd64/server/ + +RUN mkdir -p /opt/compiler/gcc-4.8.2/lib64 && \ + ln -s /lib64/ld-linux-x86-64.so.2 /opt/compiler/gcc-4.8.2/lib64/ld-linux-x86-64.so.2 && \ + mkdir -p /opt/compiler/gcc-8.2/lib64 && \ + ln -s /lib64/ld-linux-x86-64.so.2 /opt/compiler/gcc-8.2/lib64/ld-linux-x86-64.so.2 && \ + ln -sf /home/work/opbin/pbtool/bin/pblogTool3 /usr/bin/pblogTool3 && \ + ln -s /home/work/opbin/bns_tool_stub /home/work/bns_tool_stub && \ + ln -sf /home/work/opbin/bns_tool_stub/bns_tool_stub /usr/bin/get_instance_by_service && \ + mv /home/work/start.sh /start.sh + +WORKDIR /home/work/bfe/ +USER work +EXPOSE 8080 8443 8421 +ENTRYPOINT ["/start.sh"] + + diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..c3b378bf --- /dev/null +++ b/Makefile @@ -0,0 +1,96 @@ +# Copyright (c) 2021 The BFE Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# init project path +WORKROOT := $(shell pwd) +OUTDIR := $(WORKROOT)/output + +# init environment variables +export PATH := $(shell go env GOPATH)/bin:$(PATH) +export GO111MODULE := on + +# init command params +GO := go +GOBUILD := $(GO) build +GOTEST := $(GO) test +GOVET := $(GO) vet +GOGET := $(GO) get +GOGEN := $(GO) generate +GOCLEAN := $(GO) clean +GOFLAGS := -race +STATICCHECK := staticcheck + +# init arch +ARCH := $(shell getconf LONG_BIT) +ifeq ($(ARCH),64) + GOTEST += $(GOFLAGS) +endif + +# init bfe ingress version +INGRESS_VERSION ?= $(shell cat VERSION) +# init git commit id +GIT_COMMIT ?= $(shell git rev-parse HEAD) + +# init bfe ingress packages +INGRESS_PACKAGES := $(shell go list ./...) + +# make, make all +all: compile package + +# make compile, go build +compile: test build +build: + cd $(WORKROOT)/cmd/bfe_ingress_controller && GOOS=linux GOARCH=amd64 $(GOBUILD) -ldflags "-X main.version=$(INGRESS_VERSION) -X main.commit=$(GIT_COMMIT)" -o bfe_ingress_controller + +# make test, test your code +test: test-case vet-case +test-case: + $(GOTEST) -cover ./... +vet-case: + ${GOVET} ./... + +# make coverage for codecov +coverage: + echo -n > coverage.txt + for pkg in $(INGRESS_PACKAGES) ; do $(GOTEST) -coverprofile=profile.out -covermode=atomic $${pkg} && cat profile.out >> coverage.txt; done + +# make package +package: + mkdir -p $(OUTDIR) + mv $(WORKROOT)/cmd/bfe_ingress_controller $(OUTDIR)/ + cp -r $(WORKROOT)/dist/ $(OUTDIR)/ + cp $(WORKROOT)/build/adapt_bfe_docker.sh $(OUTDIR)/ + chmod a+x $(OUTDIR)/* + echo "$(GIT_COMMIT)" > $(OUTDIR)/ingress.commit + +# make check +check: + $(GO) get honnef.co/go/tools/cmd/staticcheck + $(STATICCHECK) ./... + +# make docker +docker: + docker build \ + -t bfe_ingress_controller:$(INGRESS_VERSION) \ + -f Dockerfile \ + . + +# make clean +clean: + $(GOCLEAN) + rm -rf $(OUTDIR) + rm -rf $(GOPATH)/pkg/linux_amd64 + +# avoid filename conflict and speed up build +.PHONY: all compile test package clean build \ No newline at end of file diff --git a/README.md b/README.md index 3dad6cd1..e292b8e3 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,24 @@ -# ingress-bfe -BFE Ingress Controller for Kubernetes +# BFE Ingress Controller + +BFE Ingress Controller 为基于 [BFE][] 实现的 Kubernetes [Ingress Controller][]， +用于支持在 Kubernetes 中部署 [Ingress][]。 + +## 开始使用 +参见 [部署指南文档](docs/zh_cn/deployment.md)，开始使用BFE Ingress controller。 + +## 配置文档 +参见 [配置文档](docs/zh_cn/ingress/configuration.md)，了解BFE Ingress controller更详细的配置和使用说明。 + +## 参与贡献 +* 请首先在 [Issue 列表][] 中创建一个 Issue +* 如有必要，请联系项目维护者/负责人进行进一步讨论 +* 请遵循 Golang 编程规范 +* 参见 [贡献指南文档](docs/zh_cn/contribute/how-to-contribute.md) + +## 许可 +基于 Apache 2.0 许可证，详见 [LICENSE](LICENSE) 文件说明 + +[Ingress Controller]: https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/ "Kubernetes" +[Ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ "Kubernetes" +[BFE]: https://github.com/bfenetworks/bfe "Github" +[Issue 列表]: https://github.com/bfenetworks/ingress-bfe/issues \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..49ffebca --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0-dev \ No newline at end of file diff --git a/build/adapt_bfe_docker.sh b/build/adapt_bfe_docker.sh new file mode 100644 index 00000000..a58b9678 --- /dev/null +++ b/build/adapt_bfe_docker.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -x + +# origin BFE root in docker +# see https://github.com/bfenetworks/bfe/blob/develop/Dockerfile +DOCKER_BFE_ROOT=/bfe +# new BFE root for ingress +WORK_BFE_ROOT=/home/work/bfe + +mkdir -p ${WORK_BFE_ROOT} +cp -r ${DOCKER_BFE_ROOT}/bin ${WORK_BFE_ROOT}/ +cp -r ${DOCKER_BFE_ROOT}/conf ${WORK_BFE_ROOT}/ + +# create directory for cert files +mkdir -p "${WORK_BFE_ROOT}/conf/tls_conf/certs" \ No newline at end of file diff --git a/cmd/bfe_ingress_controller/main.go b/cmd/bfe_ingress_controller/main.go new file mode 100644 index 00000000..f5a3565c --- /dev/null +++ b/cmd/bfe_ingress_controller/main.go @@ -0,0 +1,158 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "os" + "runtime" + "strings" + "time" +) + +import ( + "github.com/baidu/go-lib/log" + "github.com/baidu/go-lib/log/log4go" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/bfe_ingress" + "github.com/bfenetworks/ingress-bfe/internal/utils" +) + +var ( + help = flag.Bool("h", false, "to show help") + stdOut = flag.Bool("s", false, "to show log in stdout") + showVersion = flag.Bool("v", false, "to show version of BFE ingress controller") + showVerbose = flag.Bool("V", false, "to show verbose information") + debugLog = flag.Bool("d", false, "to show debug log (otherwise >= info)") + logPath = flag.String("l", "./log", "dir path of log") + bfeConfigRoot = flag.String("c", utils.DefaultBfeConfigRoot, "root dir path of BFE config") + reloadURLPrefix = flag.String("u", utils.DefaultReloadURLPrefix, "BFE reload URL prefix") + syncPeriod = flag.Int("p", int(utils.DefaultSyncPeriod/time.Second), + "sync period (in second) for Ingress watcher") + namespaceLabels = flag.String("f", "", "namespace label selector, split by ,") + ingressClass = flag.String("k", "", "listen ingress class name") + + namespaces utils.Namespaces +) + +var version string +var commit string + +func checkLabels(namespaces utils.Namespaces, labels string) error { + if labels == "" { + return nil + } + //namespace and label is exclusionary + if len(namespaces) > 0 && labels != "" { + return fmt.Errorf("labels and namespace sholud exclude, namespace[%s], labels[%s]", namespaces, labels) + } + labelsArr := strings.Split(labels, ",") + for _, label := range labelsArr { + keyValue := strings.Split(label, "=") + if len(keyValue) != 2 { + return fmt.Errorf("labels should be key=value, curVal[%s]", label) + } + } + return nil +} + +func main() { + flag.Var(&namespaces, "n", "namespace to watch") + flag.Parse() + if *help { + flag.PrintDefaults() + return + } + if *showVerbose { + printIngressVersion(version) + fmt.Printf("go version: %s\n", runtime.Version()) + fmt.Printf("git commit: %s\n", commit) + return + } + if *showVersion { + printIngressVersion(version) + return + } + + // check ingress parameters + if err := checkParams(); err != nil { + fmt.Printf("bfe_ingress_controller: check params error[%s]", err.Error()) + return + } + + // init log + if err := initLog(); err != nil { + fmt.Printf("bfe_ingress_controller: err in log.Init():%v\n", err) + log.Logger.Close() + os.Exit(1) + } + + labels := strings.Split(*namespaceLabels, ",") + + // create BFE Ingress controller + bfeIngress := bfe_ingress.NewBfeIngress(namespaces, labels, *ingressClass) + bfeIngress.ReloadURLPrefix = *reloadURLPrefix + bfeIngress.BfeConfigRoot = *bfeConfigRoot + bfeIngress.SyncPeriod = time.Duration(*syncPeriod) * time.Second + + // start BFE Ingress controller + log.Logger.Info("bfe_ingress_controller[version:%s] start", version) + bfeIngress.Start() + + time.Sleep(1 * time.Second) + log.Logger.Close() +} + +func initLog() error { + var logSwitch string + if *debugLog { + logSwitch = "DEBUG" + } else { + logSwitch = "INFO" + } + + log4go.SetLogBufferLength(10000) + log4go.SetLogWithBlocking(false) + log4go.SetLogFormat(log4go.FORMAT_DEFAULT_WITH_PID) + log4go.SetSrcLineForBinLog(false) + return log.Init("bfe_ingress_controller", logSwitch, *logPath, *stdOut, "midnight", 7) +} + +func checkParams() error { + if err := checkLabels(namespaces, *namespaceLabels); err != nil { + return err + } + + if *bfeConfigRoot == "" { + return fmt.Errorf("BFE config root path should not be empty") + } + + if *reloadURLPrefix == "" { + return fmt.Errorf("BFE reload URL prefix should not be empty") + } + + if *syncPeriod <= 0 { + return fmt.Errorf("sync period for Ingress watcher sholud be greater then 0, period[%d]", *syncPeriod) + } + + return nil +} + +func printIngressVersion(version string) { + fmt.Printf("bfe_ingress_controller version: %s\n", version) +} diff --git a/cmd/bfe_ingress_controller/main_test.go b/cmd/bfe_ingress_controller/main_test.go new file mode 100644 index 00000000..61265f75 --- /dev/null +++ b/cmd/bfe_ingress_controller/main_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/utils" +) + +func Test_checkLabels(t *testing.T) { + type args struct { + namespaces utils.Namespaces + labels string + } + tests := []struct { + name string + args args + wantErr bool + }{ + // TODO: Add test cases. + { + name: "check label", + args: args{ + namespaces: nil, + labels: "", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := checkLabels(tt.args.namespaces, tt.args.labels); (err != nil) != tt.wantErr { + t.Errorf("checkLabels() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/deploy/deployment.yaml b/deploy/deployment.yaml new file mode 100644 index 00000000..a9a9af1b --- /dev/null +++ b/deploy/deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bfe-ingress-controller + namespace: default + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: bfe + namespace: default + labels: + app: bfe + +spec: + replicas: 3 + selector: + matchLabels: + app: bfe + template: + metadata: + labels: + app: bfe + spec: + serviceAccountName: bfe-ingress-controller + imagePullSecrets: + - name: bfe + containers: + - name: bfe-ingress-controller + image: ${image_repo}:${version} + ports: + - name: http + containerPort: 8900 + - name: https + containerPort: 8445 + - name: monitor + containerPort: 8299 + env: + - name: INGRESS_LISTEN_NAMESPACE + value: "default" \ No newline at end of file diff --git a/deploy/ingress.yaml b/deploy/ingress.yaml new file mode 100644 index 00000000..e7a00569 --- /dev/null +++ b/deploy/ingress.yaml @@ -0,0 +1,14 @@ +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: ingress2 + namespace: default +spec: + rules: + - host: "foo.com" + http: + paths: + - path: /whoami + backend: + serviceName: whoami + servicePort: 80 \ No newline at end of file diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml new file mode 100644 index 00000000..7f40a1ac --- /dev/null +++ b/deploy/rbac.yaml @@ -0,0 +1,51 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: bfe-ingress-controller +rules: +- apiGroups: + - "" + resources: + - services + - endpoints + - secrets + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + - ingressclasses + verbs: + - get + - list + - watch + - update +- apiGroups: + - networking.k8s.io + resources: + - ingresses + - ingressclasses + verbs: + - get + - list + - watch + - update + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: bfe-ingress-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: bfe-ingress-controller +subjects: + - kind: ServiceAccount + name: bfe-ingress-controller + namespace: default \ No newline at end of file diff --git a/deploy/whoami.yaml b/deploy/whoami.yaml new file mode 100644 index 00000000..412013c8 --- /dev/null +++ b/deploy/whoami.yaml @@ -0,0 +1,41 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: whoami + namespace: default + labels: + app: containous + name: whoami + +spec: + replicas: 6 + selector: + matchLabels: + app: containous + task: whoami + template: + metadata: + labels: + app: containous + task: whoami + spec: + containers: + - name: containouswhoami + image: containous/whoami + ports: + - containerPort: 80 + +--- +apiVersion: v1 +kind: Service +metadata: + name: whoami + namespace: default + +spec: + ports: + - name: http + port: 80 + selector: + app: containous + task: whoami \ No newline at end of file diff --git a/dist/bfe.ini b/dist/bfe.ini new file mode 100644 index 00000000..75e68342 --- /dev/null +++ b/dist/bfe.ini @@ -0,0 +1,20 @@ +[program:bfe] +directory=/home/work/bfe/bin/ +command=./bfe -c ../conf -l ../log -d +priority=999 ; the relative start priority (default 999) +autostart=true ; start at supervisord start (default: true) +autorestart=true ; retstart at unexpected quit (default: true) +startsecs=10 ; number of secs prog must stay running (def. 10) +startretries=3 ; max # of serial start failures (default 3) +exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) +stopsignal=QUIT ; signal used to kill process (default TERM) +stopwaitsecs=10 ; max num secs to wait before SIGKILL (default 10) +user=work ; setuid to this UNIX account to run the program +log_stdout=true +log_stderr=true ; if true, log program stderr (def false) +logfile=/tmp/echo_time.log +logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +logfile_backups=10 ; # of logfile backups (default 10) +stdout_logfile_maxbytes=20MB ; stdout 日志文件大小，默认 50MB +stdout_logfile_backups=20 ; stdout 日志文件备份数 +stdout_logfile=/tmp/echo_time.stdout.log \ No newline at end of file diff --git a/dist/start.sh b/dist/start.sh new file mode 100644 index 00000000..a021601a --- /dev/null +++ b/dist/start.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -x + +readonly BFE_BIN=bfe + +cd /home/work/bfe/bin/ && nohup ./${BFE_BIN} -c ../conf -l ../log -d & + +if [ -n "$INGRESS_LISTEN_NAMESPACE" ]; then + cd /home/work/bfe/bin/ && ./bfe_ingress_controller -l ../log -c "/home/work/bfe/conf/" -n "$INGRESS_LISTEN_NAMESPACE" "$@" +else + cd /home/work/bfe/bin/ && ./bfe_ingress_controller -l ../log -c "/home/work/bfe/conf/" "$@" +fi \ No newline at end of file diff --git a/docs/zh_cn/README.md b/docs/zh_cn/README.md new file mode 100644 index 00000000..3c11c355 --- /dev/null +++ b/docs/zh_cn/README.md @@ -0,0 +1,8 @@ +# BFE Ingress Controller + +BFE Ingress Controller 为基于 [BFE][] 实现的 Kubernetes [Ingress Controller][]， +用于支持在 Kubernetes 中部署 [Ingress][]。 + +[Ingress Controller]: https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/ "Kubernetes" +[Ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ "Kubernetes" +[BFE]: https://github.com/bfenetworks/bfe "Github" diff --git a/docs/zh_cn/SUMMARY.md b/docs/zh_cn/SUMMARY.md new file mode 100644 index 00000000..6b241baa --- /dev/null +++ b/docs/zh_cn/SUMMARY.md @@ -0,0 +1,21 @@ +# Summary + +[comment]: <> (For user) +* [部署指南](deployment.md) + * [基于角色的控制访问（RBAC）](rbac.md) +* [配置说明](ingress/configuration.md) + * [基础配置](ingress/basic.md) + * [规则优先级](ingress/priority.md) + * [规则冲突处理](ingress/conflict.md) + * [TLS 配置](ingress/tls.md) + * [负载均衡](ingress/load-balance.md) + * [Annotation](ingress/annotation.md) +* [学习示例](example/example.md) + * [灰度发布](example/grayscale.md) +--- + +[comment]: <> (For developer) +* [参与贡献](contribute/how-to-contribute.md) + * [如何贡献代码](contribute/contribute-codes.md) + * [如何贡献文档](contribute/contribute-documents.md) + * [版本发布说明](https://www.bfe-networks.net/zh_cn/development/release_regulation/) diff --git a/docs/zh_cn/contribute/contribute-codes.md b/docs/zh_cn/contribute/contribute-codes.md new file mode 100644 index 00000000..5c326847 --- /dev/null +++ b/docs/zh_cn/contribute/contribute-codes.md @@ -0,0 +1,18 @@ +# 如何贡献代码 +## 代码要求 +- 代码注释请遵守 golang 代码规范 +- 所有代码必须具有单元测试 +- 通过所有单元测试 +- 请遵循提交代码的[一些约定](https://www.bfe-networks.net/zh_cn/development/submit_pr_guide/) +## 代码开发流程 +1. Fork +1. 克隆（Clone） +1. 创建本地分支 +1. 使用 pre-commit 钩子 +1. 编写代码 +1. 构建和测试 +1. 提交（commit） +1. 保持本地仓库最新 +1. Push 到远程仓库 + +> 可参考 BFE [开发流程](https://www.bfe-networks.net/zh_cn/development/local_dev_guide/) \ No newline at end of file diff --git a/docs/zh_cn/contribute/contribute-documents.md b/docs/zh_cn/contribute/contribute-documents.md new file mode 100644 index 00000000..0e4ace0b --- /dev/null +++ b/docs/zh_cn/contribute/contribute-documents.md @@ -0,0 +1,15 @@ +# 如何贡献文档 +## 文档要求 +- 所有内容都应该以 [Markdown][markdown] (GitHub风格)的形式编写，文件以`.md`为后缀 +- 如果是新增文档，需将新增的文档名，添加到对应的index文件中（[SUMMARY.md](../SUMMARY.md)） +## 文档开发流程 + +1. 编写文档 +1. 运行预览工具，并预览修改 + - [如何使用预览工具](https://www.bfe-networks.net/zh_cn/development/write_doc_guide/#_2) +1. 提交修改 + - 修改文档, 提交修改与PR的步骤可以参考[代码开发流程](contribute-codes.md#代码开发流程) + +> 可参考 BFE [如何贡献文档](https://www.bfe-networks.net/zh_cn/development/write_doc_guide/) + +[markdown]: https://guides.github.com/features/mastering-markdown/ \ No newline at end of file diff --git a/docs/zh_cn/contribute/how-to-contribute.md b/docs/zh_cn/contribute/how-to-contribute.md new file mode 100644 index 00000000..db8c07b8 --- /dev/null +++ b/docs/zh_cn/contribute/how-to-contribute.md @@ -0,0 +1,4 @@ +# 参与贡献 +- [如何贡献代码](contribute-codes.md) +- [如何贡献文档](contribute-documents.md) +- [版本发布说明](https://www.bfe-networks.net/zh_cn/development/release_regulation/) diff --git a/docs/zh_cn/deployment.md b/docs/zh_cn/deployment.md new file mode 100644 index 00000000..65891d73 --- /dev/null +++ b/docs/zh_cn/deployment.md @@ -0,0 +1,24 @@ +# 快速开始 + +## 安装指南 +* 部署 bfe-ingress-controller，以及相关权限配置。 + ``` shell script + kubectl apply -f deployment.yaml + kubectl apply -f rbac.yaml + ``` + - bfe-ingress-controller部署可参考 [deployment.yaml](../../deploy/deployment.yaml) 文件：修改文件中的`${image_repo}`和`${version}`，使用正确的bfe-ingress-controller镜像信息。 + + - 权限配置可参考 [rbac.yaml](../../deploy/rbac.yaml) +* 创建测试服务(例：whoami) + +* 创建ingress资源 + ``` shell script + kubectl apply -f ingress.yaml + ``` + 简单的ingess配置可参考 [ingress.yaml](../../deploy/ingress.yaml)。 + + 更多的bfe-ingress-controller所支持的Ingress配置，可参考[配置文档](ingress/configuration.md)。 + +## 权限配置文件说明 + +* [RBAC 文件编写指南](rbac.md) diff --git a/docs/zh_cn/example/example.md b/docs/zh_cn/example/example.md new file mode 100644 index 00000000..bbacd6d6 --- /dev/null +++ b/docs/zh_cn/example/example.md @@ -0,0 +1,18 @@ +# 学习示例 + +## deployment +| 程序 | 文件 | 说明 | +| ---- | ---- | ---- | +| bfe ingress controller | [deployment.yaml](../../../deploy/deployment.yaml)| 用于 bfe ingress controller 的部署| +| 示例后端服务 whoami | [whoami.yaml](../../../deploy/whoami.yaml) | 用于示例服务(whoami)的部署 | + +## ingress +| 文件 | 说明 | +| ---- | ---- | +| [ingress.yaml](../../../deploy/ingress.yaml) | 用于配置示例服务(whoami)的流量调度 | + +## rbac +| 文件 | 说明 | +| ---- | ---- | +| [rbac.yaml](../../../deploy/rbac.yaml) | 用于授予 bfe ingress controller 的权限 | + diff --git a/docs/zh_cn/example/grayscale.md b/docs/zh_cn/example/grayscale.md new file mode 100644 index 00000000..330553d3 --- /dev/null +++ b/docs/zh_cn/example/grayscale.md @@ -0,0 +1,47 @@ +# BEF-Ingress支持 Header/Cookie 灰度发布 +### 配置说明 +BFE-Ingress通过`Ingress Annotation`的方式支持`Header/Cookie`灰度发布功能，配置如下： + +```yaml +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "greyscale" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.cookie: "key: value" + bfe.ingress.kubernetes.io/router.header: "Key: Value" + +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + pathType: Exact + backend: + serviceName: service-new + servicePort: 80 +--- +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "original" + namespace: production + +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + pathType: Exact + backend: + serviceName: service-old + servicePort: 80 +``` +基于上面的配置，BFE将会 +1. 若满足 `host == example.net && path == /bar && cookie[key] == value && Header[Key] == Value`， + 则分流到`service-new`集群 +1. 否则，若仅满足 `host == example.net && path == /bar`， + 则分流到`service-old`集群 diff --git a/docs/zh_cn/ingress/annotation.md b/docs/zh_cn/ingress/annotation.md new file mode 100644 index 00000000..7cfe674f --- /dev/null +++ b/docs/zh_cn/ingress/annotation.md @@ -0,0 +1,36 @@ +# Annotation + +## 用途 +BFE Ingress Annotation 用于支持高级配配规则。 + +目前支持`Cookie`和`Header`两种，格式和优先级如下: + + +## Cookie +- 优先级：0 +``` yaml +bfe.ingress.kubernetes.io/router.cookie: "key: value" +``` +BFE将执行 `req.Cookies["Key"]==value` 的判断 + + + + +## Header +- 优先级：1 +``` yaml +bfe.ingress.kubernetes.io/router.header: "key: value" +``` +BFE将执行 `req.Headers["Key"]==value` 的判断 + + + +## 注意 +- 一个类型的Annotation下仅支持设置一个值； + ```yaml + # 例 + annotation: + bfe.ingress.kubernetes.io/router.header: "key1: value1" # 不生效 + bfe.ingress.kubernetes.io/router.header: "key2: value2" # 生效 + ``` +- 优先级数组越小，其优先级越高 \ No newline at end of file diff --git a/docs/zh_cn/ingress/basic.md b/docs/zh_cn/ingress/basic.md new file mode 100644 index 00000000..a6a4bd9e --- /dev/null +++ b/docs/zh_cn/ingress/basic.md @@ -0,0 +1,111 @@ +# Ingress 资源 + +## 什么是 Ingress 资源 +Ingress 资源定义了 Kubernetes 集群内服务对外提供服务时的流量路由规则。 +详见 [Ingress] + +## 示例 +### 简单示例 +```yaml +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: simple-ingress +spec: + rules: + - host: whoami.com + http: + paths: + - path: /testpath + pathType: Prefix + backend: + service: + name: whoami + port: + number: 80 +``` +上述 Ingress 资源定义了 1 条简单的路由规则： +若请求流量的域名为 `whoami.com`，路径前缀为 `/testpath`， +则将流量转发给`whoami` Service 的 80 端口处理 + +### 复杂示例 +```yaml +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: complex-ingress + namespace: my-namespace + annotations: + bfe.ingress.kubernetes.io/loadbalance: '{"foo": {"foo1":80, "foo2":20}}' + bfe.ingress.kubernetes.io/router.cookie: "Session: 123" + bfe.ingress.kubernetes.io/router.header: "Content-Language: zh-cn" +spec: + tls: + - hosts: + - foo.com + secretName: secret-foo-com + rules: + - host: foo.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + service: + name: foo + port: + number: 80 + - path: /bar + pathType: Exact + backend: + service: + name: bar + port: + number: 80 +``` +上述 Ingress 资源定义了 2 条复杂的路由规则，并且为 `foo.com` 配置了证书 +- 路由规则 1：若请求流量满足以下所有条件，则 +将 80%流量转发给 `foo1` Service 的 80 端口处理， +将 20%流量转发给 `foo2` Service 的 80 端口处理 + - 域名为 `foo.com` + - 路径前缀为 `/foo` + - Cookie 中，Session 值为 `123` + - Header 中，Content-Language 值为 `zh-cn` +- 路由规则 2：若请求流量满足以下所有条件，则 +将流量转发给 `bar` Service 的 80 端口处理 + - 域名为 `foo.com` + - 路径前缀为 `/bar` + - Cookie 中，Session 值为 `123` + - Header 中，Content-Language 值为 `zh-cn` + +## 路由规则组成 +- metadata + - name: Ingress 资源名 + - namespace: 应用的命名空间 + - [annotations](annotation.md) + - bfe.ingress.kubernetes.io/loadbalance: [多 Service 负载均衡](load-balance.md) 配置 + - [bfe.ingress.kubernetes.io/router.cookie](annotation.md#cookie): Cookie 匹配条件（同 Ingress 资源内共享） + - [bfe.ingress.kubernetes.io/router.header](annotation.md#header): Header 匹配条件（同 Ingress 资源内共享） +- spec + - [tls](tls.md) + - host: 证书匹配域名 + - secretName: TLS 证书 + - rules + - [host](#host): 域名匹配条件 + - http.paths + - path & [pathType](#pathtype): 路径匹配条件 + - backend.service + - name: 转发目标服务名 + - port: 转发目标服务端口 +### host +BFE Ingress 支持[前缀匹配][hostname-wildcards] + +### pathType +BFE Ingress 支持的 pathType 与 [Kubernetes 原生定义][pathType] 相近，具体为： +- Prefix: __默认__，前缀匹配 +- Exact: 精确匹配 +- ImplementationSpecific: 前缀匹配 + + [Ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/#what-is-ingress + [pathType]: https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types + [hostname-wildcards]: https://kubernetes.io/docs/concepts/services-networking/ingress/#hostname-wildcards \ No newline at end of file diff --git a/docs/zh_cn/ingress/command-line-arguments.md b/docs/zh_cn/ingress/command-line-arguments.md new file mode 100644 index 00000000..f818d99c --- /dev/null +++ b/docs/zh_cn/ingress/command-line-arguments.md @@ -0,0 +1,12 @@ +# BEF Ingress 启动参数 +bfe ingress controller支持的参数如下 + +| 选项 | 默认值 | 用途| +| --- | --- | --- | +| -n | | namespace: 设置监听的 namespace。默认监听所有的 namespace ，多个 namespace 之间用`,`分割。与 `-f`选项互斥。 | + + +示例： +```shell script +./bfe_ingress_controller -n name1,name2 +``` \ No newline at end of file diff --git a/docs/zh_cn/ingress/configuration.md b/docs/zh_cn/ingress/configuration.md new file mode 100644 index 00000000..5b6b787f --- /dev/null +++ b/docs/zh_cn/ingress/configuration.md @@ -0,0 +1,14 @@ +# 配置说明 + +## bfe ingress controller的启动参数 +bfe ingress controller 支持 [命令行参数](command-line-arguments.md)，可在 ingress controller 的部署文件中设置。 + +## Ingress 配置说明 +- [基础配置](basic.md) +- [规则优先级](priority.md) +- [规则冲突处理](conflict.md) +- [TLS 配置](tls.md) +- [负载均衡](load-balance.md) +- [Annotation](annotation.md) + - [Cookie](annotation.md#cookie) + - [Header](annotation.md#header) diff --git a/docs/zh_cn/ingress/conflict.md b/docs/zh_cn/ingress/conflict.md new file mode 100644 index 00000000..12c4393e --- /dev/null +++ b/docs/zh_cn/ingress/conflict.md @@ -0,0 +1,138 @@ +# 配置优先级 + +## 路由配置冲突 +当用户的ingress配置最终生成相同的路由规则的情况下（Host、Path、Header/Cookie完全相同）， +BFE-Ingress将按照`创建时间优先`的原则使用先配置的路由规则。 + +对于因路由冲突导致的配置生成失败，可查找相应的 ingress-controller 错误日志。 + +```yaml +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: ingress-A +spec: + rules: + - host: example.foo.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + serviceName: service1 + servicePort: 80 +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: ingress-B +spec: + rules: + - host: example.foo.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + serviceName: service2 + servicePort: 80 +#其中ingress-A先于配置ingress-B创建，则最终仅生效ingress-A。 +``` +## 跨 namespace 冲突 + +当 BFE-Ingress 在监听多个 namespace 时，如何判断是否存在冲突？如何处理？ + +以下场景可以为您解答这个问题： +* 场景 1: namespace 之间存在相同路由规则 + + 依照`创建时间优先`的原则处理，详见[路由配置冲突](#路由配置冲突) + +* 场景 2: namespace 之间存在相同命名的资源（如 ingress/service ） + + 不同 namespace 的相同资源名***不存在***冲突。 + * ingress 资源：controller 中通过 `${namespace}/${ingress}` 定位 ingress 资源； + 故不同 namespace 下的 ingress 资源无歧义，不存在冲突 + * service 资源：每个 ingress 资源 与 其中引用的 service 资源 拥有相同的 namespace 属性（默认为 default）， + controller 中通过 `${namespace}/${service}` 定位 service 资源； + 故不同 namespace 下的 service 资源无歧义，不存在冲突 + +## 状态回写 +当前Ingress的合法性是在配置生效的过程才能感知，是一个异步过程。为了能给用户反馈当前Ingress是否生效，BFE-Ingress会将Ingress的实际生效状态回写到Ingress的一个Annotation当中。 +**BFE-Ingress状态Annotation定义如下：** +```yaml +#bfe.ingress.kubernetes.io/bfe-ingress-status为BFE-Ingress预留的Annotation key， +#用于BFE-Ingress回写状态 +# status; 表示当前ingress是否合法， 取值为：success -> ingress合法， error -> ingress不合法 +# message; 当ingress不合法的情况下，message记录错误详细原因。 +bfe.ingress.kubernetes.io/bfe-ingress-status: {"status": "", "message": ""} +``` +**下面是BFE-Ingress状态回写的示例：** +`Ingress1`和`Ingress2`的路由规则完全一样(`Host:example.net, Path:/bar`)。 +```yaml +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "ingress1" + namespace: production +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service1 + servicePort: 80 +--- +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "ingress2" + namespace: production +spec: + rules: + - host: example.net + http: + paths: + - path: /foo + backend: + serviceName: service2 + servicePort: 80 +``` +根据路由冲突配置规则，`Ingress1`将生效，而`Ingress2`将被忽略。状态回写后，`Ingress1`的状态为success，而`Ingress2`的状态为fail。 +```yaml +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "ingress1" + namespace: production + annotations: + bfe.ingress.kubernetes.io/bfe-ingress-status: {"status": "success", "message": ""} +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service1 + servicePort: 80 +--- +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "ingress2" + namespace: production + annotations: + bfe.ingress.kubernetes.io/bfe-ingress-status: | + {"status": "fail", "message": "conflict with production/ingress1"} +spec: + rules: + - host: example.net + http: + paths: + - path: /foo + backend: + serviceName: service2 + servicePort: 80 +``` \ No newline at end of file diff --git a/docs/zh_cn/ingress/load-balance.md b/docs/zh_cn/ingress/load-balance.md new file mode 100644 index 00000000..56d3a312 --- /dev/null +++ b/docs/zh_cn/ingress/load-balance.md @@ -0,0 +1,24 @@ +# 多Service之间负载均衡 +BFE-Ingress通过`Ingress Annotation`的方式支持多个Service之间按权重进行负载均衡，配置如下： +```yaml +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: tls-example-ingress + annotations: + bfe.ingress.kubernetes.io/balance.weight: '{"service": {"service1":80, "service2":20}}' +spec: + tls: + - hosts: + - https-example.foo.com + secretName: testsecret-tls + rules: + - host: https-example.foo.com + http: + paths: + - path: / + pathType: Prefix + backend: + serviceName: service + servicePort: 80 +``` \ No newline at end of file diff --git a/docs/zh_cn/ingress/priority.md b/docs/zh_cn/ingress/priority.md new file mode 100644 index 00000000..48ae593c --- /dev/null +++ b/docs/zh_cn/ingress/priority.md @@ -0,0 +1,153 @@ +# 优先级说明 +当请求满足多条导流规则的情况下，BFE-Ingress会按照如下的优先级进行排序，使用优先级最高的导流规则： +- 优先满足域名匹配； +- 域名相同的场景下，优先满足更精确的路径匹配； +- 域名、路径相同的场景下，优先满足匹配条件更多的场景； +- 域名、路径、匹配条件个数相同情况下，按照匹配条件的固定顺序确定优先级； + - Cookie的优先级高于Header； + +## 优先级示例 +### 域名优先 +```yaml +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "host_priority1" + namespace: production + +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service1 + servicePort: 80 +--- +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "host_priority2" + namespace: production + +spec: + rules: + - host: example2.net + http: + paths: + - path: /bar + backend: + serviceName: service2 + servicePort: 80 +``` +针对`curl "http://example.net/bar"`优先匹配规则`host_priority1` + +### 域名相同，优先路径 +```yaml +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "path_priority1" + namespace: production + +spec: + rules: + - host: example.net + http: + paths: + - path: /bar/foo + backend: + serviceName: service1 + servicePort: 80 +--- +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "path_priority2" + namespace: production + bfe.ingress.kubernetes.io/router.header: "key: value" +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service2 + servicePort: 80 +``` +针对`curl "http://example.net/bar/foo" -H "Key: value"`优先匹配规则`path_priority1` + +### 路径优先规则个数 +```yaml +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "cond_priority1" + namespace: production + bfe.ingress.kubernetes.io/router.header: "key: value" +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service1 + servicePort: 80 +--- +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "cond_priority1" + namespace: production +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service2 + servicePort: 80 +``` +针对`curl "http://example.net/bar/foo" -H "Key: value"`优先匹配规则`cond_priority1` + +### 规则个数相同，按固定顺序排序 +```yaml +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "multi_cond_priority1" + namespace: production + bfe.ingress.kubernetes.io/router.header: "header-key: value" +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service1 + servicePort: 80 +--- +kind: Ingress +apiVersion: networking.k8s.io/v1beta1 +metadata: + name: "multi_cond_priority2" + namespace: production + bfe.ingress.kubernetes.io/router.cookie: "cookie-key: value" +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service2 + servicePort: 80 +``` +例如当前BFE-Ingress中`Cookie`的优先级高于`Header`的优先级。 +针对`curl "http://example.net/bar/foo" -H "Header-key: value" --cookie "cookie-key: value"`优先匹配规则`multi_cond_priority2` + diff --git a/docs/zh_cn/ingress/tls.md b/docs/zh_cn/ingress/tls.md new file mode 100644 index 00000000..b6b44ef7 --- /dev/null +++ b/docs/zh_cn/ingress/tls.md @@ -0,0 +1,35 @@ +# TLS 配置 +原生Ingress的TLS中，证书和密钥是通过Secrets进行保存，例子如下： +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: testsecret-tls + namespace: default +data: + tls.crt: base64 encoded cert + tls.key: base64 encoded key +type: kubernetes.io/tls +``` +Ingress配置 +```yaml +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: tls-example-ingress +spec: + tls: + - hosts: + - https-example.foo.com + secretName: testsecret-tls + rules: + - host: https-example.foo.com + http: + paths: + - path: / + pathType: Prefix + backend: + serviceName: service1 + servicePort: 80 +``` +BFE-Ingress按照同样的方式来管理TLS的证书和密钥，其余更高级的TLS功能，包括一些TLS的配置，密钥加密等功能，需要参考BFE-Ingress CRD来实现； diff --git a/docs/zh_cn/rbac.md b/docs/zh_cn/rbac.md new file mode 100644 index 00000000..8de0591a --- /dev/null +++ b/docs/zh_cn/rbac.md @@ -0,0 +1,44 @@ +# 基于角色的控制访问（RBAC） + +## 总览 + +此示例适用于在启用了RBAC的环境中部署的 bfe-ingres-controller + +基于角色的访问控制由四层组成: + +1. `ClusterRole` - 分配给适用于整个集群的角色的权限 +2. `ClusterRoleBinding` - 将ClusterRole绑定到特定帐户 +3. `Role` - 分配给适用于特定名称空间的角色的权限 +4. `RoleBinding` - 将角色绑定到特定帐户 + +为了将RBAC应用于`bfe-ingres-controller`，应将该控制器分配给`ServiceAccount`。 +该`ServiceAccount`应该绑定到为`bfe-ingres-controller`定义的`Roles`和`ClusterRoles` + +## 示例说明 + +### 创建 ServiceAccount + +在此 [示例](../../deploy/deployment.yaml) 中，创建了一个 ServiceAccount ，即`bfe-ingres-controller`。 + +### 创建权限集 + +在此 [示例](../../deploy/rbac.yaml) 中定义了 1 组权限: +- 由名为`bfe-ingres-controller`的`ClusterRole`定义的集群范围权限， + +#### 集群权限 + +授予这些权限是为了使 bfe-ingres-controller 能够充当跨集群的入口。 +这些权限被授予名为`bfe-ingres-controller`的 ClusterRole + +- `services`, `endpoints`, `secrets`, `namespaces`: get, list, watch +- `ingresses`, `ingressclasses`: get, list, watch, update + +如果在启动bfe-ingres-controller时覆盖了两个参数，请进行相应调整 + +### 权限绑定 + +在此 [示例](../../deploy/rbac.yaml) 中，ServiceAccount `bfe-ingres-controller` 绑定到 ClusterRole `bfe-ingres-controller`。 + +!!! 注意：[deployment](../../deploy/deployment.yaml) 中 +- 容器关联的 serviceAccountName 必须与 serviceAccount 匹配。 +- metadata，容器参数 和 POD_NAMESPACE 中的 namespace 应位于对应 ingress namespace 中 \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..6fbaefb4 --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module github.com/bfenetworks/ingress-bfe + +go 1.14 + +require ( + bou.ke/monkey v1.0.2 + github.com/baidu/go-lib v0.0.0-20200819072111-21df249f5e6a + github.com/bfenetworks/bfe v1.2.1-0.20210625051839-e9fbe8ca0423 + github.com/mitchellh/hashstructure v1.1.0 + github.com/stretchr/testify v1.7.0 + golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect + k8s.io/api v0.19.2 + k8s.io/apimachinery v0.19.2 + k8s.io/client-go v0.19.2 + k8s.io/utils v0.0.0-20200729134348-d5654de09c73 +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..6fab5e80 --- /dev/null +++ b/go.sum @@ -0,0 +1,481 @@ +bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= +bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/abbot/go-http-auth v0.4.1-0.20181019201920-860ed7f246ff h1:9ZqcMQ0fB+ywKACVjGfZM4C7Uq9D5rq0iSmwIjX187k= +github.com/abbot/go-http-auth v0.4.1-0.20181019201920-860ed7f246ff/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= +github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asergeyev/nradix v0.0.0-20170505151046-3872ab85bb56 h1:Wi5Tgn8K+jDcBYL+dIMS1+qXYH2r7tpRAyBgqrWfQtw= +github.com/asergeyev/nradix v0.0.0-20170505151046-3872ab85bb56/go.mod h1:8BhOLuqtSuT5NZtZMwfvEibi09RO3u79uqfHZzfDTR4= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/baidu/go-lib v0.0.0-20200819072111-21df249f5e6a h1:m/u39GNhkoUSC9WxTuM5hWShEqEfVioeXDiqiQd6tKg= +github.com/baidu/go-lib v0.0.0-20200819072111-21df249f5e6a/go.mod h1:FneHDqz3wLeDGdWfRyW4CzBbCwaqesLGIFb09N80/ww= +github.com/bfenetworks/bfe v1.2.1-0.20210625051839-e9fbe8ca0423 h1:Lj1pkx79YvC0C20//gr8yajbeFGfxpsXFwxBD/gWHPw= +github.com/bfenetworks/bfe v1.2.1-0.20210625051839-e9fbe8ca0423/go.mod h1:InpmkIed4Q4IySPGHoLBBD/l2m+56BX7GLOBfel0tz4= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chris-ramon/douceur v0.2.0 h1:IDMEdxlEUUBYBKE4z/mJnFyVXox+MjuEVDJNN27glkU= +github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpRXQKjTR8nIBE= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/elastic/go-sysinfo v1.1.1 h1:ZVlaLDyhVkDfjwPGU55CQRCRolNpc7P0BbyhhQZQmMI= +github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 h1:IPJ3dvxmJ4uczJe5YQdrYB16oTJlGSC/OyZDqUk9xX4= +github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869/go.mod h1:cJ6Cj7dQo+O6GJNiMx+Pa94qKj+TG8ONdKHgMNIyyag= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/microcosm-cc/bluemonday v1.0.3 h1:EjVH7OqbU219kdm8acbveoclh2zZFqPJTJw6VUlTLAQ= +github.com/microcosm-cc/bluemonday v1.0.3/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w= +github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg= +github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= +github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5 h1:ZCnq+JUrvXcDVhX/xRolRBZifmabN1HcS1wrPSvxhrU= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2 h1:nY8Hti+WKaP0cRsSeQ026wU03QsM762XBeCXBb9NAWI= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug= +github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng= +github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls= +github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tjfoc/gmsm v1.3.2 h1:7JVkAn5bvUJ7HtU08iW6UiD+UTmJTIToHCfeFzkcCxM= +github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= +github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= +github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= +github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zmap/go-iptree v0.0.0-20170831022036-1948b1097e25 h1:LRoXAcKX48QV4LV23W5ZtsG/MbJOgNUNvWiXwM0iLWw= +github.com/zmap/go-iptree v0.0.0-20170831022036-1948b1097e25/go.mod h1:qOasALtPByO1Jk6LhgpNv6htPMK2QJfiGorUk57nO/U= +go.elastic.co/apm v1.7.2 h1:0nwzVIPp4PDBXSYYtN19+1W5V+sj+C25UjqxDVoKcA8= +go.elastic.co/apm v1.7.2/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0= +go.elastic.co/apm/module/apmhttp v1.7.2 h1:2mRh7SwBuEVLmJlX+hsMdcSg9xaielCLElaPn/+i34w= +go.elastic.co/apm/module/apmhttp v1.7.2/go.mod h1:sTFWiWejnhSdZv6+dMgxGec2Nxe/ZKfHfz/xtRM+cRY= +go.elastic.co/apm/module/apmot v1.7.2 h1:FXvTXGvVOwc26K3llgPdxenWoPv9VdO5CQ3aAfc5lZY= +go.elastic.co/apm/module/apmot v1.7.2/go.mod h1:VD2nUkebUPrP1hqIarimIEsoM9xyuK0lO83fCx6l/Z8= +go.elastic.co/fastjson v1.0.0 h1:ooXV/ABvf+tBul26jcVViPT3sBir0PvXgibYB1IQQzg= +go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0= +go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1 h1:/7cs52RnTJmD43s3uxzlq2U7nqVTd/37viQwMrMNlOM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/square/go-jose.v2 v2.4.1 h1:H0TmLt7/KmzlrDOpa1F+zr0Tk90PbJYBfsVUmRLrf9Y= +gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= +k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= +k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc= +k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= +k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/bfe_ingress/ingress.go b/internal/bfe_ingress/ingress.go new file mode 100644 index 00000000..bc516741 --- /dev/null +++ b/internal/bfe_ingress/ingress.go @@ -0,0 +1,155 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bfe_ingress + +import ( + "os" + "sync" + "syscall" + "time" +) + +import ( + "github.com/baidu/go-lib/log" + "github.com/bfenetworks/bfe/bfe_util/signal_table" + networking "k8s.io/api/networking/v1beta1" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/builder" + "github.com/bfenetworks/ingress-bfe/internal/kubernetes_client" +) + +const ( + statusStaring = iota + statusFailed + statusStarted +) + +type ingressList []*networking.Ingress + +type BfeIngress struct { + namespaces []string + labels []string + ingressClass string + + stopCh chan struct{} + + status int // status for staring process + wg sync.WaitGroup // wait group for graceful exit + + BfeConfigRoot string // BFE config root path, for config dumping + ReloadURLPrefix string // common prefix for BFE reload URL + SyncPeriod time.Duration // period for ingress watcher to re-sync +} + +func NewBfeIngress(namespaces, labels []string, ingressClass string) *BfeIngress { + return &BfeIngress{ + namespaces: namespaces, + labels: labels, + ingressClass: ingressClass, + stopCh: make(chan struct{}), + status: statusStaring, + } +} + +func (ing *BfeIngress) Start() { + ingressesCh := make(chan ingressList, 1) + client, err := kubernetes_client.NewKubernetesClient() + if err != nil { + log.Logger.Warn("error in NewKubernetesClient(): %s", err) + } + + ing.initSignalTable() + ing.startWatcher(client, ingressesCh, ing.SyncPeriod) + ing.startProcessor(client, ingressesCh) + + if ing.status == statusFailed { + // exit if failed in somewhere + ing.Shutdown(nil) + } else { + // update status as started + ing.status = statusStarted + } + + // waiting for shutdown + ing.wg.Wait() + log.Logger.Info("stop ingress") +} + +// start ingress processor goroutine +func (ing *BfeIngress) startProcessor(client *kubernetes_client.KubernetesClient, ingressesCh chan ingressList) { + // skip when ingress failed for some reason + if ing.status == statusFailed { + return + } + + // create processor + processor, err := NewProcessor(client, ingressesCh, ing.stopCh) + if err != nil { + log.Logger.Error(err) + ing.status = statusFailed + return + } + processor.dumper = builder.NewDumper(ing.BfeConfigRoot) + processor.reloader = builder.NewReloader(ing.ReloadURLPrefix) + + // start processor goroutine + ing.wg.Add(1) + go processor.Start(&ing.wg) +} + +// start ingress watcher goroutine +func (ing *BfeIngress) startWatcher(client *kubernetes_client.KubernetesClient, ingressesCh chan ingressList, + syncPeriod time.Duration) { + // skip when ingress failed for some reason + if ing.status == statusFailed { + return + } + + // create watcher + watcher, err := NewWatcher(ing.namespaces, ing.labels, ing.ingressClass, client, ingressesCh, ing.stopCh) + if err != nil { + log.Logger.Error(err) + ing.status = statusFailed + return + } + watcher.syncPeriod = syncPeriod + + // start watcher + ing.wg.Add(1) + go watcher.Start(&ing.wg) +} + +// shutdown ingress by broadcasting stop signal +func (ing *BfeIngress) Shutdown(sig os.Signal) { + close(ing.stopCh) +} + +func (ing *BfeIngress) initSignalTable() { + /* create signal table */ + signalTable := signal_table.NewSignalTable() + + /* register signal handlers */ + signalTable.Register(syscall.SIGQUIT, ing.Shutdown) + signalTable.Register(syscall.SIGTERM, signal_table.TermHandler) + signalTable.Register(syscall.SIGHUP, signal_table.IgnoreHandler) + signalTable.Register(syscall.SIGILL, signal_table.IgnoreHandler) + signalTable.Register(syscall.SIGTRAP, signal_table.IgnoreHandler) + signalTable.Register(syscall.SIGABRT, signal_table.IgnoreHandler) + + /* start signal handler routine */ + signalTable.StartSignalHandle() +} diff --git a/internal/bfe_ingress/ingress_status.go b/internal/bfe_ingress/ingress_status.go new file mode 100644 index 00000000..05cf610d --- /dev/null +++ b/internal/bfe_ingress/ingress_status.go @@ -0,0 +1,68 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package bfe_ingress + +import ( + "encoding/json" + "fmt" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/builder" + "github.com/bfenetworks/ingress-bfe/internal/kubernetes_client" +) + +var ( + StatusAnnotationKey = fmt.Sprintf("%s%s", builder.BfeAnnotationPrefix, "bfe-ingress-status") +) + +type StatusMsg struct { + Status string `json:"status"` + Message string `json:"message"` +} + +type IngressStatusWriter struct { + client *kubernetes_client.KubernetesClient +} + +func (w *IngressStatusWriter) SetError(namespace, name, msg string) error { + errMsg := w.getErrorMsg(msg) + w.client.UpdateIngressAnnotation(namespace, name, StatusAnnotationKey, errMsg) + return nil +} + +func (w *IngressStatusWriter) SetSuccess(namespace, name string) error { + msg := w.getSuccessMsg("") + w.client.UpdateIngressAnnotation(namespace, name, StatusAnnotationKey, msg) + return nil +} + +func (w *IngressStatusWriter) getErrorMsg(msg string) string { + var status = StatusMsg{ + Status: "error", + Message: msg, + } + jsons, _ := json.Marshal(status) + + return string(jsons) +} + +func (w *IngressStatusWriter) getSuccessMsg(msg string) string { + var status = StatusMsg{ + Status: "success", + Message: msg, + } + jsons, _ := json.Marshal(status) + return string(jsons) +} diff --git a/internal/bfe_ingress/ingress_status_test.go b/internal/bfe_ingress/ingress_status_test.go new file mode 100644 index 00000000..72069c74 --- /dev/null +++ b/internal/bfe_ingress/ingress_status_test.go @@ -0,0 +1,89 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bfe_ingress + +import ( + "testing" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/kubernetes_client" +) + +func TestIngressStatusWriter_getErrorMsg(t *testing.T) { + type fields struct { + client *kubernetes_client.KubernetesClient + } + type args struct { + msg string + } + tests := []struct { + name string + fields fields + args args + want string + }{ + // TODO: Add test cases. + { + name: "normal", + fields: fields{client: nil}, + args: args{msg: "error msg"}, + want: "{\"status\":\"error\",\"message\":\"error msg\"}", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := &IngressStatusWriter{ + client: tt.fields.client, + } + if got := w.getErrorMsg(tt.args.msg); got != tt.want { + t.Errorf("IngressStatusWriter.getErrorMsg() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIngressStatusWriter_getSuccessMsg(t *testing.T) { + type fields struct { + client *kubernetes_client.KubernetesClient + } + type args struct { + msg string + } + tests := []struct { + name string + fields fields + args args + want string + }{ + // TODO: Add test cases. + { + name: "normal", + fields: fields{client: nil}, + args: args{msg: "success msg"}, + want: "{\"status\":\"success\",\"message\":\"success msg\"}", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := &IngressStatusWriter{ + client: tt.fields.client, + } + if got := w.getSuccessMsg(tt.args.msg); got != tt.want { + t.Errorf("IngressStatusWriter.getSuccessMsg() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/bfe_ingress/processer.go b/internal/bfe_ingress/processer.go new file mode 100644 index 00000000..ba581cda --- /dev/null +++ b/internal/bfe_ingress/processer.go @@ -0,0 +1,190 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bfe_ingress + +import ( + "fmt" + "sort" + "sync" + "time" +) + +import ( + "github.com/baidu/go-lib/log" + networking "k8s.io/api/networking/v1beta1" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/builder" + "github.com/bfenetworks/ingress-bfe/internal/kubernetes_client" + "github.com/bfenetworks/ingress-bfe/internal/utils" +) + +type Processor struct { + client *kubernetes_client.KubernetesClient + + ingressCh chan ingressList + stopCh chan struct{} + + dumper *builder.Dumper // for dumping BFE config + reloader *builder.Reloader // for reloading BFE config + + statusWriter *IngressStatusWriter +} + +func (p *Processor) initBfeConfigBuilder() []builder.BfeConfigBuilder { + version := "reload" + var builders []builder.BfeConfigBuilder + + builders = append(builders, builder.NewBfeBalanceConfigBuilder(p.client, version, p.dumper, p.reloader)) + builders = append(builders, builder.NewBfeRouteConfigBuilder(p.client, version, p.dumper, p.reloader)) + builders = append(builders, builder.NewBfeTLSConfigBuilder(p.client, version, p.dumper, p.reloader)) + + return builders +} + +func (p *Processor) processIngresses(ingresses ingressList) { + cur := time.Now().UTC().String() + + builders := p.initBfeConfigBuilder() + + for _, ingress := range ingresses { + log.Logger.Info("time[%s] ingress: namespaces[%s], ingress[%s], stamp[%s]", cur, ingress.Namespace, ingress.Name, ingress.CreationTimestamp.Time.String()) + + var submittedBuilders = make([]builder.BfeConfigBuilder, 0) + unSubmitted := false + + // submit current ingress to different type of config builders + for _, builder := range builders { + err := builder.Submit(ingress) + if err != nil { + log.Logger.Warn("namespaces[%s] ingress[%s] submit error[%s]", ingress.Namespace, ingress.Name, err.Error()) + unSubmitted = true + p.doRollback(submittedBuilders, ingress) + p.setStatus(ingress, true, err.Error()) + break + } else { + submittedBuilders = append(submittedBuilders, builder) + } + } + if !unSubmitted { + p.setStatus(ingress, false, "") + } + } + if err := p.build(builders); err != nil { + return + } + if err := p.dump(builders); err != nil { + return + } + if err := p.reload(builders); err != nil { + return + } +} + +func (p *Processor) build(builders []builder.BfeConfigBuilder) error { + for _, builder := range builders { + err := builder.Build() + if err != nil { + log.Logger.Warn("builder build error[%s]", err.Error()) + return err + } + } + return nil +} + +func (p *Processor) dump(builders []builder.BfeConfigBuilder) error { + for _, builder := range builders { + err := builder.Dump() + if err != nil { + log.Logger.Warn("builder dump error[%s]", err.Error()) + return err + } + } + return nil +} + +func (p *Processor) reload(builders []builder.BfeConfigBuilder) error { + for _, builder := range builders { + err := builder.Reload() + if err != nil { + log.Logger.Warn("builder reload error[%s]", err.Error()) + return err + } + } + return nil +} + +// when route config conflict, the older config will win, so sort ingress by create time +func (p *Processor) sortIngresses(ingresses ingressList) { + sort.Slice(ingresses, func(i, j int) bool { + if ingresses[i].CreationTimestamp.Equal(&ingresses[j].CreationTimestamp) { + return ingresses[i].Name < ingresses[j].Name + } + return ingresses[i].CreationTimestamp.Before(&ingresses[j].CreationTimestamp) + }) +} + +func (p *Processor) Start(wg *sync.WaitGroup) { + defer wg.Done() + + for { + select { + case ingresses := <-p.ingressCh: + log.Logger.Info("process [%d] ingress", len(ingresses)) + p.sortIngresses(ingresses) + p.processIngresses(ingresses) + case <-p.stopCh: + log.Logger.Info("stop processor") + return + } + } +} + +func NewProcessor(c *kubernetes_client.KubernetesClient, ingressCh chan ingressList, + stopCh chan struct{}) (*Processor, error) { + + // check parameters + if c == nil || ingressCh == nil || stopCh == nil { + return nil, fmt.Errorf("create processor fail") + } + + return &Processor{ + client: c, + ingressCh: ingressCh, + stopCh: stopCh, + statusWriter: &IngressStatusWriter{client: c}, + dumper: builder.NewDumper(utils.DefaultBfeConfigRoot), + reloader: builder.NewReloader(utils.DefaultReloadURLPrefix), + }, nil +} + +func (p *Processor) doRollback(builders []builder.BfeConfigBuilder, ingress *networking.Ingress) { + for _, builder := range builders { + log.Logger.Debug("rollback namespaces[%s] ingress[%s]", ingress.Namespace, ingress.Name) + err := builder.Rollback(ingress) + if err != nil { + log.Logger.Warn("namespaces[%s] ingress[%s] submit error[%s]", ingress.Namespace, ingress.Name, err.Error()) + } + } +} + +func (p *Processor) setStatus(ingress *networking.Ingress, err bool, msg string) { + if err { + p.statusWriter.SetError(ingress.Namespace, ingress.Name, msg) + } else { + p.statusWriter.SetSuccess(ingress.Namespace, ingress.Name) + } +} diff --git a/internal/bfe_ingress/watcher.go b/internal/bfe_ingress/watcher.go new file mode 100644 index 00000000..fb67f32e --- /dev/null +++ b/internal/bfe_ingress/watcher.go @@ -0,0 +1,137 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bfe_ingress + +import ( + "fmt" + "reflect" + "sort" + "sync" + "time" +) + +import ( + "github.com/baidu/go-lib/log" + "github.com/mitchellh/hashstructure" + core "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/kubernetes_client" + "github.com/bfenetworks/ingress-bfe/internal/utils" +) + +type IngressWatcher struct { + namespace []string + labels []string + ingressClass string + + client *kubernetes_client.KubernetesClient + syncPeriod time.Duration // period for ingress watcher to re-sync + ingressCh chan ingressList + stopCh chan struct{} +} + +var IngressService map[string]bool // ingress services watched + +// NewWatcher creates watcher for ingress in k8s +func NewWatcher(namespaces []string, labels []string, ingressClass string, + client *kubernetes_client.KubernetesClient, ingressCh chan ingressList, + stopCh chan struct{}) (*IngressWatcher, error) { + + // check parameters + if client == nil || ingressCh == nil || stopCh == nil { + return nil, fmt.Errorf("create ingress watcher fail") + } + + return &IngressWatcher{ + namespace: namespaces, + labels: labels, + ingressClass: ingressClass, + + client: client, + syncPeriod: utils.DefaultSyncPeriod, + ingressCh: ingressCh, + stopCh: stopCh, + }, nil +} + +func (iw *IngressWatcher) hash(ingressList []*networking.Ingress) (uint64, error) { + cpIngressList := make([]*networking.Ingress, 0) + for _, ingress := range ingressList { + cpIngress := ingress.DeepCopy() + if (*cpIngress).Annotations != nil { + delete((*cpIngress).Annotations, StatusAnnotationKey) + log.Logger.Info("name{%s} annotations{%v} spec.rules{%v}", (*cpIngress).Name, (*cpIngress).Annotations, (*cpIngress).Spec.Rules) + } + (*cpIngress).ObjectMeta.ResourceVersion = "" + cpIngressList = append(cpIngressList, cpIngress) + } + sort.Slice(cpIngressList, func(i, j int) bool { + if cpIngressList[i].CreationTimestamp.Equal(&cpIngressList[j].CreationTimestamp) { + return cpIngressList[i].Name < cpIngressList[j].Name + } + return cpIngressList[i].CreationTimestamp.Before(&cpIngressList[j].CreationTimestamp) + }) + return hashstructure.Hash(cpIngressList, nil) +} + +func (iw *IngressWatcher) Start(wg *sync.WaitGroup) { + defer wg.Done() + + IngressService = make(map[string]bool) + eventCh := iw.client.Watch(iw.namespace, iw.labels, iw.ingressClass, iw.syncPeriod) + for { + select { + case msg := <-eventCh: + t := reflect.TypeOf(msg).String() + log.Logger.Debug("eventCh type is %s, eventCh message is %+v", t, msg) + switch t { + case "*v1beta1.Ingress": + log.Logger.Info("process ingress resource") + data := (msg).(*networking.Ingress) + parseServiceFromIngress(data) + log.Logger.Info("ingress services info: %v", IngressService) + + case "*v1.Endpoints": + log.Logger.Info("process endpoints resource") + data := (msg).(*core.Endpoints) + endService := fmt.Sprintf("%s:%s", data.Namespace, data.Name) + if _, ok := IngressService[endService]; !ok { + continue + } + } + + ingresses := iw.client.GetIngresses() + iw.ingressCh <- ingresses + + case <-iw.stopCh: + log.Logger.Info("stop watcher") + return + } + } +} + +func parseServiceFromIngress(ingress *networking.Ingress) { + for _, rule := range ingress.Spec.Rules { + for _, path := range rule.HTTP.Paths { + serviceName := path.Backend.ServiceName + if serviceName != "" { + IngressService[fmt.Sprintf("%s:%s", ingress.Namespace, serviceName)] = true + } + } + } +} diff --git a/internal/builder/annotation.go b/internal/builder/annotation.go new file mode 100644 index 00000000..6b459881 --- /dev/null +++ b/internal/builder/annotation.go @@ -0,0 +1,196 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package builder + +import ( + "encoding/json" + "fmt" + "sort" + "strings" +) + +const ( + BfeAnnotationPrefix = "bfe.ingress.kubernetes.io/" + CookieKey = "router.cookie" + HeaderKey = "router.header" + + LoadBalanceWeightKey = "balance.weight" + LoadBalanceWeightAnnotation = BfeAnnotationPrefix + LoadBalanceWeightKey + + CookiePriority = 0 + HeaderPriority = 1 +) + +type BfeAnnotation interface { + Priority() int + Check() error + Build() string +} + +type cookieAnnotation struct { + annotationStr string +} + +func (cookie *cookieAnnotation) Priority() int { + return CookiePriority +} + +// Check validates cookie annotation string with form "key: value" +func (cookie *cookieAnnotation) Check() error { + if len(strings.Split(cookie.annotationStr, ":")) < 2 { + return fmt.Errorf("invalid cookie annotation str") + } + return nil +} + +func (cookie *cookieAnnotation) Build() string { + strs := strings.Split(cookie.annotationStr, ":") + key := strs[0] + key = strings.TrimSpace(key) + value := strings.Join(strs[1:], ":") + value = strings.TrimSpace(value) + con := fmt.Sprintf("req_cookie_value_in(\"%s\", \"%v\", false)", key, value) + return con +} + +type headerAnnotation struct { + annotationStr string +} + +func (header *headerAnnotation) Priority() int { + return HeaderPriority +} + +// Check validates header annotation string with form "key: value" +func (header *headerAnnotation) Check() error { + if len(strings.Split(header.annotationStr, ":")) < 2 { + return fmt.Errorf("invalid header annotation str") + } + return nil +} + +func (header *headerAnnotation) Build() string { + strs := strings.Split(header.annotationStr, ":") + key := strs[0] + key = strings.TrimSpace(key) + value := strings.Join(strs[1:], ":") + value = strings.TrimSpace(value) + con := fmt.Sprintf("req_header_value_in(\"%s\", \"%v\", false)", key, value) + return con +} + +// BuildBfeAnnotations extract sorted BFE annotations for k8s annotations +func BuildBfeAnnotations(k8sAnnotations map[string]string) []BfeAnnotation { + bfeAnnotations := make([]BfeAnnotation, 0) + + // build annotations + for key, value := range k8sAnnotations { + bfeAnnotation, err := BuildBfeAnnotation(key, value) + if err == nil { + bfeAnnotations = append(bfeAnnotations, bfeAnnotation) + } + } + + // sort annotations + SortAnnotations(bfeAnnotations) + return bfeAnnotations +} + +// BuildBfeAnnotation build BFE annotation by key & value +func BuildBfeAnnotation(key string, value string) (BfeAnnotation, error) { + if !strings.HasPrefix(key, BfeAnnotationPrefix) { + return nil, fmt.Errorf("Unsupported annotation: %s", key) + } + newKey := strings.ReplaceAll(key, BfeAnnotationPrefix, "") + var annotation BfeAnnotation + switch newKey { + case CookieKey: + annotation = &cookieAnnotation{annotationStr: value} + case HeaderKey: + annotation = &headerAnnotation{annotationStr: value} + default: + return nil, fmt.Errorf("Unsupported annotation: %s", newKey) + } + if err := annotation.Check(); err != nil { + return nil, err + } + return annotation, nil +} + +func SortAnnotations(annotationConds []BfeAnnotation) { + sort.Slice(annotationConds, func(i, j int) bool { + return annotationConds[i].Priority() < annotationConds[j].Priority() + }) +} + +type ServicesWeight map[string]int +type LoadBalance map[string]ServicesWeight + +func (l *LoadBalance) ContainService(service string) bool { + if l == nil || (*l) == nil { + return false + } + _, ok := (*l)[service] + return ok +} + +func (l *LoadBalance) GetService(serviceName string) (ServicesWeight, error) { + if !l.ContainService(serviceName) { + return nil, fmt.Errorf("load balance donot contain[%s]", serviceName) + } + return (*l)[serviceName], nil +} + +func BuildLoadBalanceAnnotation(key string, value string) (LoadBalance, error) { + if key != LoadBalanceWeightAnnotation { + return nil, fmt.Errorf("Unsupported annotation: %s", key) + } + var lb = make(LoadBalance) + err := json.Unmarshal([]byte(value), &lb) + if err != nil { + return nil, err + } + + for _, services := range lb { + sum := 0 + var tmpList = make([]string, 0) + for name, weight := range services { + if weight < 0 { + return nil, fmt.Errorf("weight of load balance service should greate than or equal to zero") + } + sum += weight + tmpList = append(tmpList, name) + } + //add sort make it easy to do unit test + sort.Slice(tmpList, func(i, j int) bool { + return tmpList[i] < tmpList[j] + }) + if sum == 0 { + return nil, fmt.Errorf("sum of load balance service weight is zero") + } + if sum != 100 { + curSum := 0 + for index, name := range tmpList { + weight := services[name] + newWeight := int(float32(weight)/float32(sum)*100.0 + 0.5) + if index == len(tmpList)-1 { + newWeight = 100 - curSum + } + curSum += newWeight + services[name] = newWeight + } + } + } + return lb, nil +} diff --git a/internal/builder/annotation_test.go b/internal/builder/annotation_test.go new file mode 100644 index 00000000..5102f030 --- /dev/null +++ b/internal/builder/annotation_test.go @@ -0,0 +1,498 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package builder + +import ( + "reflect" + "testing" +) + +func Test_cookieAnnotation_Build(t *testing.T) { + type fields struct { + annotationStr string + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "cookie build", + fields: fields{ + annotationStr: "Cookie-Test: Test", + }, + want: "req_cookie_value_in(\"Cookie-Test\", \"Test\", false)", + }, + { + name: "cookie build value with space", + fields: fields{ + annotationStr: "Cookie-Test: Test ", + }, + want: "req_cookie_value_in(\"Cookie-Test\", \"Test\", false)", + }, + { + name: "cookie build key with space", + fields: fields{ + annotationStr: "Cookie-Test : Test ", + }, + want: "req_cookie_value_in(\"Cookie-Test\", \"Test\", false)", + }, + { + name: "cookie build key with space", + fields: fields{ + annotationStr: "Cookie-Test: Test:123 ", + }, + want: "req_cookie_value_in(\"Cookie-Test\", \"Test:123\", false)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cookie := &cookieAnnotation{ + annotationStr: tt.fields.annotationStr, + } + if got := cookie.Build(); got != tt.want { + t.Errorf("cookieAnnotation.Build() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_cookieAnnotation_Check(t *testing.T) { + type fields struct { + annotationStr string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "check normal", + fields: fields{ + annotationStr: "Key: value", + }, + wantErr: false, + }, + { + name: "check normal", + fields: fields{ + annotationStr: "Key value", + }, + wantErr: true, + }, + { + name: "check normal", + fields: fields{ + annotationStr: "Key: value:123", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cookie := &cookieAnnotation{ + annotationStr: tt.fields.annotationStr, + } + if err := cookie.Check(); (err != nil) != tt.wantErr { + t.Errorf("cookieAnnotation.Check() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_headerAnnotation_Build(t *testing.T) { + type fields struct { + annotationStr string + } + tests := []struct { + name string + fields fields + want string + }{ + // TODO: Add test cases. + { + name: "header build", + fields: fields{ + annotationStr: "Header-Test: Test", + }, + want: "req_header_value_in(\"Header-Test\", \"Test\", false)", + }, + { + name: "header build value with space", + fields: fields{ + annotationStr: "Header-Test: Test ", + }, + want: "req_header_value_in(\"Header-Test\", \"Test\", false)", + }, + { + name: "header build key with space", + fields: fields{ + annotationStr: "Header-Test : Test ", + }, + want: "req_header_value_in(\"Header-Test\", \"Test\", false)", + }, + { + name: "header build key with space", + fields: fields{ + annotationStr: "Header-Test: Test:123 ", + }, + want: "req_header_value_in(\"Header-Test\", \"Test:123\", false)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + header := &headerAnnotation{ + annotationStr: tt.fields.annotationStr, + } + if got := header.Build(); got != tt.want { + t.Errorf("headerAnnotation.Build() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_headerAnnotation_Check(t *testing.T) { + type fields struct { + annotationStr string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + // TODO: Add test cases. + { + name: "check normal", + fields: fields{ + annotationStr: "Key: value", + }, + wantErr: false, + }, + { + name: "check normal", + fields: fields{ + annotationStr: "Key value", + }, + wantErr: true, + }, + { + name: "check normal", + fields: fields{ + annotationStr: "Key: value:123", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + header := &headerAnnotation{ + annotationStr: tt.fields.annotationStr, + } + if err := header.Check(); (err != nil) != tt.wantErr { + t.Errorf("headerAnnotation.Check() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestBuildBfeAnnotation(t *testing.T) { + type args struct { + key string + value string + } + tests := []struct { + name string + args args + want BfeAnnotation + wantErr bool + }{ + // TODO: Add test cases. + { + name: "build header", + args: args{ + key: "bfe.ingress.kubernetes.io/router.header", + value: "Header: Value", + }, + want: &headerAnnotation{annotationStr: "Header: Value"}, + wantErr: false, + }, + { + name: "build cookie", + args: args{ + key: "bfe.ingress.kubernetes.io/router.cookie", + value: "Cookie: Value", + }, + want: &cookieAnnotation{annotationStr: "Cookie: Value"}, + wantErr: false, + }, + { + name: "build err", + args: args{ + key: "bfe.ingress.kubernetes.io/router.err", + value: "Header: Value", + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := BuildBfeAnnotation(tt.args.key, tt.args.value) + if (err != nil) != tt.wantErr { + t.Errorf("BuildBfeAnnotation() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("BuildBfeAnnotation() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSortAnnotations(t *testing.T) { + type args struct { + annotationConds []BfeAnnotation + } + tests := []struct { + name string + args args + want []BfeAnnotation + }{ + // TODO: Add test cases. + { + name: "sort normal", + args: args{ + annotationConds: []BfeAnnotation{ + &headerAnnotation{}, + &cookieAnnotation{}, + }, + }, + want: []BfeAnnotation{ + &cookieAnnotation{}, + &headerAnnotation{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SortAnnotations(tt.args.annotationConds) + }) + } +} + +func TestBuildLoadBalanceAnnotation(t *testing.T) { + type args struct { + key string + value string + } + tests := []struct { + name string + args args + want LoadBalance + wantErr bool + }{ + // TODO: Add test cases. + { + name: "error key", + args: args{ + key: "error key", + value: `{"service": {"service": 100}}`, + }, + want: nil, + wantErr: true, + }, + { + name: "normal", + args: args{ + key: LoadBalanceWeightAnnotation, + value: `{"service": {"service": 100}, "service2":{"service2-1": 33, "service2-2":66}}`, + }, + want: LoadBalance{ + "service": map[string]int{"service": 100}, + "service2": map[string]int{"service2-1": 33, "service2-2": 67}, + }, + wantErr: false, + }, + { + name: "abnormal json", + args: args{ + key: LoadBalanceWeightAnnotation, + value: `{"service": {"service": 100}, "service2":{"service2-1": 33, "service2-2":"66"}}`, + }, + want: nil, + wantErr: true, + }, + { + name: "normal", + args: args{ + key: LoadBalanceWeightAnnotation, + value: `{"service": {"service": 100}, "service2":{"service2-1": 33, "service2-2":66}, "service3":{"service3-1": 33, "service3-2":33, "service3-3":33}}`, + }, + want: LoadBalance{ + "service": map[string]int{"service": 100}, + "service2": map[string]int{"service2-1": 33, "service2-2": 67}, + "service3": map[string]int{"service3-1": 33, "service3-2": 33, "service3-3": 34}, + }, + wantErr: false, + }, + { + name: "sum zero", + args: args{ + key: LoadBalanceWeightAnnotation, + value: `{"service": {"service": 100}, "service2":{"service2-1": 33, "service2-2":66}, "service3":{"service3-1": 0, "service3-2":0, "service3-3":0}}`, + }, + want: nil, + wantErr: true, + }, + { + name: "diff normal", + args: args{ + key: LoadBalanceWeightAnnotation, + value: `{"service": {"service": 100}, "service2":{"service2-1": 33, "service2-2":66}, "service3":{"service3-1": 1, "service3-2":2, "service3-3":100000}}`, + }, + want: LoadBalance{ + "service": map[string]int{"service": 100}, + "service2": map[string]int{"service2-1": 33, "service2-2": 67}, + "service3": map[string]int{"service3-1": 0, "service3-2": 0, "service3-3": 100}, + }, + wantErr: false, + }, + { + name: "diff normal 1 1 1", + args: args{ + key: LoadBalanceWeightAnnotation, + value: `{"service": {"service": 100}, "service2":{"service2-1": 33, "service2-2":66}, "service3":{"service3-1": 1, "service3-2":1, "service3-3":1}}`, + }, + want: LoadBalance{ + "service": map[string]int{"service": 100}, + "service2": map[string]int{"service2-1": 33, "service2-2": 67}, + "service3": map[string]int{"service3-1": 33, "service3-2": 33, "service3-3": 34}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := BuildLoadBalanceAnnotation(tt.args.key, tt.args.value) + if (err != nil) != tt.wantErr { + t.Errorf("BuildLoadBalanceAnnotation() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("BuildLoadBalanceAnnotation() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLoadBalance_GetService(t *testing.T) { + val := `{"service": {"service": 100}, "service2":{"service2-1": 33, "service2-2":66}, "service3":{"service3-1": 33, "service3-2":33, "service3-3":33}}` + l, _ := BuildLoadBalanceAnnotation(LoadBalanceWeightAnnotation, val) + type args struct { + serviceName string + } + tests := []struct { + name string + l *LoadBalance + args args + want ServicesWeight + wantErr bool + }{ + // TODO: Add test cases. + { + name: "normal", + l: &l, + args: args{ + serviceName: "service", + }, + want: map[string]int{ + "service": 100, + }, + wantErr: false, + }, + { + name: "normal", + l: &l, + args: args{ + serviceName: "service3", + }, + want: map[string]int{ + "service3-1": 33, + "service3-2": 33, + "service3-3": 34, + }, + wantErr: false, + }, + { + name: "normal", + l: &l, + args: args{ + serviceName: "service4", + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.l.GetService(tt.args.serviceName) + if (err != nil) != tt.wantErr { + t.Errorf("LoadBalance.GetService() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("LoadBalance.GetService() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLoadBalance_ContainService(t *testing.T) { + val := `{"service": {"service": 100}, "service2":{"service2-1": 33, "service2-2":66}, "service3":{"service3-1": 33, "service3-2":33, "service3-3":33}}` + l, _ := BuildLoadBalanceAnnotation(LoadBalanceWeightAnnotation, val) + type args struct { + service string + } + tests := []struct { + name string + l *LoadBalance + args args + want bool + }{ + // TODO: Add test cases. + { + name: "normal", + l: &l, + args: args{ + service: "service", + }, + want: true, + }, + { + name: "normal", + l: &l, + args: args{ + service: "service4", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.l.ContainService(tt.args.service); got != tt.want { + t.Errorf("LoadBalance.ContainService() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/builder/balance_builder.go b/internal/builder/balance_builder.go new file mode 100644 index 00000000..4b559654 --- /dev/null +++ b/internal/builder/balance_builder.go @@ -0,0 +1,356 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package builder + +import ( + "fmt" + "sort" + "strconv" +) + +import ( + "github.com/bfenetworks/bfe/bfe_config/bfe_cluster_conf/cluster_table_conf" + "github.com/bfenetworks/bfe/bfe_config/bfe_cluster_conf/gslb_conf" + core "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/kubernetes_client" +) + +const ( + ConfigNameBalanceConf = "gslb_data_conf" + + GslbData = "cluster_conf/gslb.data" + ClusterTableData = "cluster_conf/cluster_table.data" +) + +type BfeBalanceConf struct { + gslbConf *gslb_conf.GslbConf + clusterTableConf *cluster_table_conf.ClusterTableConf +} + +type ingressSubCluster struct { + name string + port string + weight int +} + +type ingressSubClusters struct { + subClusters []*ingressSubCluster + refCount int +} + +type ingressClusters map[string]*ingressSubClusters + +type BfeBalanceConfigBuilder struct { + client *kubernetes_client.KubernetesClient + + dumper *Dumper + reloader *Reloader + + version string + hostName string + + balanceConf BfeBalanceConf + + clusters ingressClusters +} + +func NewBfeBalanceConfigBuilder(client *kubernetes_client.KubernetesClient, version string, dumper *Dumper, reloader *Reloader) *BfeBalanceConfigBuilder { + c := &BfeBalanceConfigBuilder{} + c.client = client + c.version = version + c.dumper = dumper + c.reloader = reloader + c.hostName = "bfe-ingress-controller" + c.clusters = make(ingressClusters) + return c +} + +func (c *BfeBalanceConfigBuilder) submitClusters(clusterName string, subClusterList []*ingressSubCluster) error { + if _, ok := c.clusters[clusterName]; !ok { + c.clusters[clusterName] = &ingressSubClusters{ + subClusters: subClusterList, + refCount: 1, + } + } else { + c.clusters[clusterName].refCount++ + } + return nil +} + +func (c *BfeBalanceConfigBuilder) rollbackClusters(clusterName string) error { + if _, ok := c.clusters[clusterName]; !ok { + return nil + } else { + c.clusters[clusterName].refCount-- + if c.clusters[clusterName].refCount == 0 { + delete(c.clusters, clusterName) + } + } + return nil +} + +func (c *BfeBalanceConfigBuilder) Submit(ingress *networking.Ingress) error { + var err error + + // parse load-balance parameters from annotation + var balance LoadBalance + for key, value := range ingress.Annotations { + if key == LoadBalanceWeightAnnotation { + balance, err = BuildLoadBalanceAnnotation(key, value) + if err != nil { + return err + } + break + } + } + + type cacheItem struct { + clusterName string + subCluster []*ingressSubCluster + } + var cache = make([]cacheItem, 0) + + for _, rule := range ingress.Spec.Rules { + for _, p := range rule.HTTP.Paths { + if !balance.ContainService(p.Backend.ServiceName) { + clusterName := SingleClusterName(ingress.Namespace, p.Backend.ServiceName) + subClusterName := p.Backend.ServiceName + + eps, err := c.client.GetEndpoints(ingress.Namespace, p.Backend.ServiceName) + if err != nil { + return fmt.Errorf("[%s/Services/%s] get endpoints error: %s", + ingress.Namespace, p.Backend.ServiceName, err.Error()) + } + if len(eps.Subsets) == 0 { + return fmt.Errorf("[%s/Services/%s] has no backend", ingress.Namespace, p.Backend.ServiceName) + } + + subClusterObj := ingressSubCluster{ + name: subClusterName, + port: p.Backend.ServicePort.StrVal, + weight: 100, + } + cache = append(cache, cacheItem{clusterName, []*ingressSubCluster{&subClusterObj}}) + + } else { + clusterName := MultiClusterName(ingress.Namespace, ingress.Name, p.Backend.ServiceName) + subClusters, _ := balance.GetService(p.Backend.ServiceName) + + ingressSubList := make([]*ingressSubCluster, 0) + for subClusterName, weight := range subClusters { + eps, err := c.client.GetEndpoints(ingress.Namespace, subClusterName) + if err != nil { + return fmt.Errorf("[%s/Services/%s] get endpoints error: %s", + ingress.Namespace, p.Backend.ServiceName, err.Error()) + } + if len(eps.Subsets) == 0 { + return fmt.Errorf("[%s/Services/%s] has no backend", ingress.Namespace, p.Backend.ServiceName) + } + if weight < 0 { + return fmt.Errorf("[%s/Services/%s] invalid weight %d, less than zero", + ingress.Namespace, subClusterName, weight) + } + subClusterObj := ingressSubCluster{ + name: subClusterName, + port: p.Backend.ServicePort.StrVal, + weight: weight, + } + ingressSubList = append(ingressSubList, &subClusterObj) + } + cache = append(cache, cacheItem{clusterName, ingressSubList}) + } + } + } + for _, item := range cache { + c.submitClusters(item.clusterName, item.subCluster) + } + return nil +} + +func (c *BfeBalanceConfigBuilder) Rollback(ingress *networking.Ingress) error { + var balance LoadBalance + var err error + for key, value := range ingress.Annotations { + if key == LoadBalanceWeightAnnotation { + balance, err = BuildLoadBalanceAnnotation(key, value) + if err != nil { + return err + } + break + } + } + + for _, rule := range ingress.Spec.Rules { + for _, p := range rule.HTTP.Paths { + var clusterName string + if !balance.ContainService(p.Backend.ServiceName) { + clusterName = SingleClusterName(ingress.Namespace, p.Backend.ServiceName) + } else { + clusterName = MultiClusterName(ingress.Namespace, ingress.Name, p.Backend.ServiceName) + } + err := c.rollbackClusters(clusterName) + if err != nil { + return fmt.Errorf("Rollback ingress error: %s", err.Error()) + } + } + } + return nil +} + +func (c *BfeBalanceConfigBuilder) Build() error { + clusterBackend, err := c.buildAllClusterBackend() + if err != nil { + return err + } + + gslbCluster, err := c.buildGslbConf() + if err != nil { + return err + } + + c.balanceConf = BfeBalanceConf{ + clusterTableConf: &cluster_table_conf.ClusterTableConf{ + Config: &clusterBackend, + Version: &c.version, + }, + gslbConf: &gslb_conf.GslbConf{ + Clusters: &gslbCluster, + Ts: &c.version, + Hostname: &c.hostName, + }, + } + return nil +} + +func (c *BfeBalanceConfigBuilder) buildGslbConf() (gslb_conf.GslbClustersConf, error) { + gslbClustersConf := make(gslb_conf.GslbClustersConf) + + for clusterName, subClusters := range c.clusters { + gslbClusterConf := make(gslb_conf.GslbClusterConf) + for _, subCluster := range (*subClusters).subClusters { + gslbClusterConf[subCluster.name] = subCluster.weight + } + gslbClustersConf[clusterName] = gslbClusterConf + } + + return gslbClustersConf, nil +} + +func (c *BfeBalanceConfigBuilder) buildAllClusterBackend() (cluster_table_conf.AllClusterBackend, error) { + allClusterBackend := make(cluster_table_conf.AllClusterBackend) + + for clusterName, subClusters := range c.clusters { + for _, subCluster := range (*subClusters).subClusters { + clusterBackend, err := c.buildClusterBackend(Namespace(clusterName), (*subCluster).name, (*subCluster).port) + if err != nil { + return allClusterBackend, err + } + if _, ok := allClusterBackend[clusterName]; !ok { + allClusterBackend[clusterName] = make(cluster_table_conf.ClusterBackend) + } + for subClusterName, val := range clusterBackend { + allClusterBackend[clusterName][subClusterName] = val + } + } + } + return allClusterBackend, nil +} + +func (c *BfeBalanceConfigBuilder) buildClusterBackend(namespace, serviceName string, port string) (cluster_table_conf.ClusterBackend, error) { + var clusterBackend cluster_table_conf.ClusterBackend + + eps, err := c.client.GetEndpoints(namespace, serviceName) + + if err != nil { + return clusterBackend, err + } + subClusterBackend := buildSubClusterBackend(eps, port) + clusterBackend = make(cluster_table_conf.ClusterBackend) + + sort.Slice(subClusterBackend, func(i, j int) bool { + return *subClusterBackend[i].Name > *subClusterBackend[j].Name + }) + + if len(subClusterBackend) == 0 { + return clusterBackend, fmt.Errorf("[%s/Services/%s] has no endpoints", namespace, serviceName) + } + + clusterBackend[serviceName] = subClusterBackend + return clusterBackend, nil +} + +func buildSubClusterBackend(eps *core.Endpoints, port string) cluster_table_conf.SubClusterBackend { + var subClusterBackend cluster_table_conf.SubClusterBackend + defaultWeight := 1 + for _, subsets := range eps.Subsets { + backend := buildBackend(subsets, port, defaultWeight) + subClusterBackend = append(subClusterBackend, backend...) + } + return subClusterBackend +} + +// buildBackend builds backend for given subsets with port and weight +func buildBackend(subsets core.EndpointSubset, port string, weight int) cluster_table_conf.SubClusterBackend { + var subClusterBackend cluster_table_conf.SubClusterBackend + for _, addr := range subsets.Addresses { + if port != "" { + name := fmt.Sprintf("%s:%s", addr.IP, port) + ip := addr.IP + portVal, _ := strconv.Atoi(port) + backendConf := cluster_table_conf.BackendConf{ + Name: &name, + Addr: &ip, + Port: &portVal, + Weight: &weight, + } + subClusterBackend = append(subClusterBackend, &backendConf) + } else { + for _, setPort := range subsets.Ports { + name := fmt.Sprintf("%s:%d", addr.IP, setPort.Port) + portVal := int(setPort.Port) + ip := addr.IP + backendConf := cluster_table_conf.BackendConf{ + Name: &name, + Addr: &ip, + Port: &portVal, + Weight: &weight, + } + subClusterBackend = append(subClusterBackend, &backendConf) + } + } + } + return subClusterBackend +} + +func (c *BfeBalanceConfigBuilder) Dump() error { + err := c.dumper.DumpJson(c.balanceConf.gslbConf, GslbData) + if err != nil { + return fmt.Errorf("dump gslb.data error: %v", err) + } + + err = c.dumper.DumpJson(c.balanceConf.clusterTableConf, ClusterTableData) + if err != nil { + return fmt.Errorf("dump cluster_table.data error: %v", err) + } + + return nil +} + +func (c *BfeBalanceConfigBuilder) Reload() error { + return c.reloader.DoReload(c.balanceConf, ConfigNameBalanceConf) +} diff --git a/internal/builder/balance_builder_test.go b/internal/builder/balance_builder_test.go new file mode 100644 index 00000000..3f51190d --- /dev/null +++ b/internal/builder/balance_builder_test.go @@ -0,0 +1,155 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* test for balance builder*/ +package builder + +import ( + "reflect" + "testing" +) + +import ( + "bou.ke/monkey" + "github.com/stretchr/testify/assert" + core "k8s.io/api/core/v1" +) + +import ( + k8s "github.com/bfenetworks/ingress-bfe/internal/kubernetes_client" +) + +var ( + testServices = map[string]service{ + "service1": { + Endpoints: []string{"172.0.0.1"}, + Port: 8081, + }, + "service2": { + Endpoints: []string{"172.0.0.2"}, + Port: 8082, + }, + } +) + +// service defines its endpoints for testing +type service struct { + Endpoints []string + Port int32 +} + +func TestBfeBalanceConfigBuilder_Build(t *testing.T) { + testCases := map[string]interface{}{ + "single": map[string]interface{}{ + "annotation": map[string]interface{}{ + "load_balance": TestBfeBalanceConfigBuilder_Build_CaseLoadBalance, + "other": TestBfeBalanceConfigBuilder_Build_CaseNoLoadBalance, + }, + }, + } + + traverseTestCases(t, testCases) + monkey.UnpatchAll() +} + +// mock function for k8s.KubernetesClient.GetEndpoints() +func mockGetEndpoints(_ *k8s.KubernetesClient, namespace, name string) (*core.Endpoints, error) { + service, ok := testServices[name] + if !ok { + return nil, nil + } + + addresses := make([]core.EndpointAddress, 0) + for _, endpoint := range service.Endpoints { + address := core.EndpointAddress{IP: endpoint} + addresses = append(addresses, address) + } + return &core.Endpoints{ + Subsets: []core.EndpointSubset{ + { + Addresses: addresses, + Ports: []core.EndpointPort{{Port: service.Port}}, + }, + }, + }, nil +} + +func TestBfeBalanceConfigBuilder_Build_CaseLoadBalance(t *testing.T) { + b, err := balanceConfigBuilderGenerator("single/annotation/load_balance") + if err != nil { + t.Fatalf("balanceConfigBuilderGenerator(%s): %s", + "single/annotation/load_balance", err) + } + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + assert.NotNil(t, b.balanceConf.clusterTableConf, "clusterTableConf is empty") + gslbConf := b.balanceConf.gslbConf + assert.NotNil(t, gslbConf, "gslbConf is empty") + assert.NotNil(t, gslbConf.Clusters, "GSLB clusters is empty") + assert.Equal(t, 1, len(*gslbConf.Clusters)) + t.Logf("clusterTableConf: %s", jsonify(b.balanceConf.clusterTableConf)) + t.Logf("gslbConf: %s", jsonify(b.balanceConf.gslbConf)) +} + +func TestBfeBalanceConfigBuilder_Build_CaseNoLoadBalance(t *testing.T) { + b, err := balanceConfigBuilderGenerator("single/annotation/other") + if err != nil { + t.Fatalf("balanceConfigBuilderGenerator(%s): %s", + "single/annotation/load_balance", err) + } + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + assert.NotNil(t, b.balanceConf.clusterTableConf, "clusterTableConf is empty") + gslbConf := b.balanceConf.gslbConf + assert.NotNil(t, gslbConf, "gslbConf is empty") + assert.NotNil(t, gslbConf.Clusters, "GSLB clusters is empty") + assert.Equal(t, 2, len(*gslbConf.Clusters)) + t.Logf("clusterTableConf: %s", jsonify(b.balanceConf.clusterTableConf)) + t.Logf("gslbConf: %s", jsonify(b.balanceConf.gslbConf)) +} + +// balanceConfigBuilderGenerator generate balance config builder from file +// Params: +// name: file name prefix +// Returns: +// *BfeBalanceConfigBuilder: builder generated by non-conflicting ingresses +// error: error for last conflict/wrong ingress +func balanceConfigBuilderGenerator(name string) (*BfeBalanceConfigBuilder, error) { + client := &k8s.KubernetesClient{} + monkey.PatchInstanceMethod(reflect.TypeOf(client), "GetEndpoints", mockGetEndpoints) + + // load ingress from file + ingresses := loadIngress(name) + + // submit ingress to builder + var submitErr error + builder := NewBfeBalanceConfigBuilder(client, "0", nil, nil) + for _, ingress := range ingresses { + err := builder.Submit(ingress) + if err != nil { + submitErr = err + } + } + return builder, submitErr +} diff --git a/internal/builder/builder.go b/internal/builder/builder.go new file mode 100644 index 00000000..c5575550 --- /dev/null +++ b/internal/builder/builder.go @@ -0,0 +1,63 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builder + +import ( + networking "k8s.io/api/networking/v1beta1" +) + +// BfeConfigCache is interface for cacheable BfeConfigBuilder, +// which can cache ingresses one-by-one for specific BFE config +type BfeConfigCache interface { + /* + Submit an ingress resource to cache of specific BFE config. + + All ingress resources will be submitted in sequence. + It supposed that: Submit with error won't change BfeConfigCache. + */ + Submit(ingress *networking.Ingress) error + + /* + Rollback is reverse operation of Submit. + It changes cache of BFE config with a submitted ingress resource, as if it hadn't been submitted. + */ + Rollback(ingress *networking.Ingress) error +} + +// BfeConfigDumper dumps specific BFE config +type BfeConfigDumper interface { + Dump() error +} + +// BfeConfigReloader reloads specific BFE config +// usually work with BfeConfigDumper( reload after dump) +type BfeConfigReloader interface { + Reload() error +} + +// BfeConfigBuilder build specific BFE config +type BfeConfigBuilder interface { + // cache information from ingresses + BfeConfigCache + + // Build builds BFE config, usually use information cached before + Build() error + + // dump BFE config for subsequent use (e.g. reloading, troubleshooting ...) + BfeConfigDumper + + // reload BFE config + BfeConfigReloader +} diff --git a/internal/builder/cluster_name.go b/internal/builder/cluster_name.go new file mode 100644 index 00000000..6b70ea1f --- /dev/null +++ b/internal/builder/cluster_name.go @@ -0,0 +1,48 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package builder + +import ( + "fmt" + "strings" +) + +import ( + networking "k8s.io/api/networking/v1beta1" +) + +func ClusterName(ingress *networking.Ingress, balance LoadBalance, p networking.HTTPIngressPath) string { + if !balance.ContainService(p.Backend.ServiceName) { + return SingleClusterName(ingress.Namespace, p.Backend.ServiceName) + } + + return MultiClusterName(ingress.Namespace, ingress.Name, p.Backend.ServiceName) +} + +// SingleClusterName return cluster name for single k8s service +// e.g. "default_whoAmI" +func SingleClusterName(namespace, serviceName string) string { + return fmt.Sprintf("%s_%s", namespace, serviceName) +} + +// MultiClusterName return cluster name for multi k8s service +// e.g. "default_ingressTest_whoAmI" +func MultiClusterName(namespace, ingressName, serviceKey string) string { + return fmt.Sprintf("%s_%s_%s", namespace, ingressName, serviceKey) +} + +// Namespace return namespace which parsed from cluster name +func Namespace(clusterName string) string { + return strings.Split(clusterName, "_")[0] +} diff --git a/internal/builder/dump.go b/internal/builder/dump.go new file mode 100644 index 00000000..411cf919 --- /dev/null +++ b/internal/builder/dump.go @@ -0,0 +1,60 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builder + +import ( + "io/ioutil" + "os" + "path" +) + +import ( + "github.com/bfenetworks/bfe/bfe_util" +) + +const ( + FilePerm os.FileMode = 0744 +) + +type Dumper struct { + dumpRoot string // root path for dump +} + +func NewDumper(root string) *Dumper { + return &Dumper{ + dumpRoot: root, + } +} + +// DumpJson dumps json object to a relative path from dump root +func (d *Dumper) DumpJson(jsonObject interface{}, relativePath string) error { + absolutePath := path.Join(d.dumpRoot, relativePath) + return bfe_util.DumpJson(jsonObject, absolutePath, FilePerm) +} + +// DumpBytes dumps byte data to a relative path from dump root +func (d *Dumper) DumpBytes(data []byte, relativePath string) error { + absolutePath := path.Join(d.dumpRoot, relativePath) + return ioutil.WriteFile(absolutePath, data, FilePerm) +} + +// Join joins dump root and any number of suffix path elements into a single path, +// separating them with slashes. Empty elements are ignored. +// The result is Cleaned. +func (d *Dumper) Join(suffixes ...string) string { + elem := []string{d.dumpRoot} + elem = append(elem, suffixes...) + return path.Join(elem[:]...) +} diff --git a/internal/builder/path_type.go b/internal/builder/path_type.go new file mode 100644 index 00000000..10ad86c7 --- /dev/null +++ b/internal/builder/path_type.go @@ -0,0 +1,32 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package builder + +import ( + networking "k8s.io/api/networking/v1beta1" +) + +func BfePathType(pathType *networking.PathType) networking.PathType { + if pathType == nil { + return networking.PathTypePrefix + } + + switch *pathType { + // see: https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types + case networking.PathTypeExact: + return networking.PathTypeExact + default: + return networking.PathTypePrefix + } +} diff --git a/internal/builder/reload.go b/internal/builder/reload.go new file mode 100644 index 00000000..87781a26 --- /dev/null +++ b/internal/builder/reload.go @@ -0,0 +1,104 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builder + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "reflect" + "sync" +) + +import ( + "github.com/baidu/go-lib/log" +) + +const ( + BodyLimit = 1024 +) + +type ConfigCache struct { + lastConfigMap map[string]interface{} + lock sync.Mutex +} + +func (c *ConfigCache) isConfigUpdated(key string, val interface{}) bool { + c.lock.Lock() + defer c.lock.Unlock() + lastConfig, ok := c.lastConfigMap[key] + if !ok { + return true + } + return !reflect.DeepEqual(lastConfig, val) +} + +func (c *ConfigCache) setConfig(key string, val interface{}) { + c.lock.Lock() + c.lastConfigMap[key] = val + c.lock.Unlock() +} + +type Reloader struct { + urlPrefix string + cache *ConfigCache +} + +func NewReloader(prefix string) *Reloader { + return &Reloader{ + urlPrefix: prefix, + cache: &ConfigCache{ + lastConfigMap: make(map[string]interface{}), + lock: sync.Mutex{}, + }, + } +} + +func (r *Reloader) DoReload(newConfig interface{}, configName string) error { + if !r.cache.isConfigUpdated(configName, newConfig) { + return nil + } + log.Logger.Info("config[%s] has diff should reload", configName) + r.cache.setConfig(configName, newConfig) + r.reloadBfe(configName) + return nil +} + +func (r *Reloader) reloadBfe(configName string) error { + // reload BFE + url := r.urlPrefix + configName + res, err := http.Get(url) + if err != nil { + return err + } + defer res.Body.Close() + + // reload succeed + if res.StatusCode == http.StatusOK { + return nil + } + + // reload fail + // parse reason from response body + failReason, err := ioutil.ReadAll(io.LimitReader(res.Body, BodyLimit)) + if err != nil { + err = fmt.Errorf("failed to reload %s, and parse fail reason error: %s", configName, err) + } else { + err = fmt.Errorf("failed to reload %s: %s", configName, failReason) + } + log.Logger.Warn(err) + return err +} diff --git a/internal/builder/route_builder.go b/internal/builder/route_builder.go new file mode 100644 index 00000000..d4414c43 --- /dev/null +++ b/internal/builder/route_builder.go @@ -0,0 +1,683 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package builder + +import ( + "fmt" + "path" + "sort" + "strings" +) + +import ( + "github.com/bfenetworks/bfe/bfe_config/bfe_cluster_conf/cluster_conf" + "github.com/bfenetworks/bfe/bfe_config/bfe_route_conf/host_rule_conf" + "github.com/bfenetworks/bfe/bfe_config/bfe_route_conf/route_rule_conf" + networking "k8s.io/api/networking/v1beta1" + "k8s.io/utils/pointer" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/kubernetes_client" +) + +const ( + DefaultProduct = "default" + ConfigNameRouteConf = "server_data_conf" + + UnknownConditionType = -1 + + ConditionTypeContainExactHostExactPath = iota + ConditionTypeContainExactHostPrefixPath + ConditionTypeContainOnlyExactHost + ConditionTypeContainWildcardHostExactPath + ConditionTypeContainWildcardHostPrefixPath + ConditionTypeContainOnlyWildcardHost + ConditionTypeContainOnlyExactPath + ConditionTypeContainOnlyPrefixPath + ConditionTypeContainNoHostPath +) + +const ( + ClusterNameAdvancedMode = "ADVANCED_MODE" +) + +const ( + HostTypeExact = iota + HostTypeWildcard + HostTypeNoRestriction +) + +type BfeRouteConf struct { + hostTableConf *host_rule_conf.HostTableConf + routeTableFile *route_rule_conf.RouteTableFile + bfeClusterConf *cluster_conf.BfeClusterConf +} + +type ingressRawRuleInfo struct { + Host string + Path string + PathType *networking.PathType + Annotations []BfeAnnotation +} + +type ingressRouteRuleFile struct { + RouteRuleFile route_rule_conf.AdvancedRouteRuleFile + RawRuleInfo ingressRawRuleInfo + ConditionType int +} + +const ( + ServerDataConfDir = "server_data_conf/" + + HostRuleData = ServerDataConfDir + "host_rule.data" + RouteRuleData = ServerDataConfDir + "route_rule.data" + ClusterConfData = ServerDataConfDir + "cluster_conf.data" +) + +type ingressRecordRule struct { + rule *ingressRouteRuleFile + ingress *networking.Ingress +} + +// record condition str -> ingress record rule +type Rule map[string]*ingressRecordRule + +// record host-> route config of this host +type HostRule map[string]Rule + +// Get rule by host and condition +func (r HostRule) Get(host, condition string) (*ingressRecordRule, bool) { + conditionRule, ok := r[host] + if !ok { + return nil, false + } + + if rule, ok := conditionRule[condition]; !ok { + return nil, false + } else { + return rule, true + } +} + +// DisposableSet sets rule for host and condition, set only success once for same host and condition +// previous set value will returned when failed +func (r HostRule) DisposableSet(host, condition string, value *ingressRecordRule) (*ingressRecordRule, bool) { + conditionRule, ok := r[host] + if !ok { + conditionRule = make(Rule) + r[host] = conditionRule + } + + if rule, ok := conditionRule[condition]; !ok { + conditionRule[condition] = value + return nil, true + } else { + return rule, false + } +} + +type BfeRouteConfigBuilder struct { + client *kubernetes_client.KubernetesClient + + dumper *Dumper + reloader *Reloader + version string + + routeConf BfeRouteConf + + rules HostRule +} + +// advancedRuleCoverage to record host & path of current advanced rules +type advancedRuleCoverage struct { + HostPath map[string]map[string]bool +} + +func NewBfeRouteConfigBuilder(client *kubernetes_client.KubernetesClient, version string, dumper *Dumper, r *Reloader) *BfeRouteConfigBuilder { + c := &BfeRouteConfigBuilder{} + c.client = client + c.version = version + c.dumper = dumper + c.reloader = r + c.rules = make(HostRule) + return c +} + +func (c *BfeRouteConfigBuilder) Submit(ingress *networking.Ingress) error { + // build balance from annotation + var balance LoadBalance + var err error + for key, value := range ingress.Annotations { + if key == LoadBalanceWeightAnnotation { + balance, err = BuildLoadBalanceAnnotation(key, value) + if err != nil { + return err + } + break + } + } + + // generate rules in cache + var cacheHostRule = make(HostRule) + annotationConds := BuildBfeAnnotations(ingress.Annotations) + for _, rule := range ingress.Spec.Rules { + for _, p := range rule.HTTP.Paths { + product := rule.Host + if product == "" { + product = DefaultProduct + } + cond, conditionType := buildCondition(rule.Host, p.Path, p.PathType, annotationConds) + + // check conflict with previous rules in previous ingress + if conflictRule, ok := c.rules.Get(product, cond); ok { + conflictIngress := conflictRule.ingress + return fmt.Errorf("route cond conflict, ingress[%s/%s] ingored cause other ingress[%s/%s]", + ingress.Namespace, ingress.Name, conflictIngress.Namespace, conflictIngress.Name) + } + + // generate rule and add to cache + ruleRecord := recordRule(ingress, rule, cond, conditionType, balance, annotationConds, p) + if conflictRule, ok := cacheHostRule.DisposableSet(product, cond, ruleRecord); !ok { + conflictIngress := conflictRule.ingress + return fmt.Errorf("route cond conflict, ingress[%s/%s] ingored cause other ingress[%s/%s]", + ingress.Namespace, ingress.Name, conflictIngress.Namespace, conflictIngress.Name) + } + } + } + + // save rules from cache to ConfigBuilder + for product, productRule := range cacheHostRule { + for cond, rule := range productRule { + c.rules.DisposableSet(product, cond, rule) + } + } + + return nil +} + +func recordRule(ingress *networking.Ingress, rule networking.IngressRule, cond string, conditionType int, + balance LoadBalance, annotations []BfeAnnotation, p networking.HTTPIngressPath) *ingressRecordRule { + clusterName := ClusterName(ingress, balance, p) + ruleFile := ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &cond, + ClusterName: &clusterName, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: rule.Host, + Path: p.Path, + PathType: p.PathType, + Annotations: annotations, + }, + ConditionType: conditionType, + } + ingressRecordRule := &ingressRecordRule{ + rule: &ruleFile, + ingress: ingress, + } + return ingressRecordRule +} + +func (c *BfeRouteConfigBuilder) Rollback(ingress *networking.Ingress) error { + annotationConds := BuildBfeAnnotations(ingress.Annotations) + + for _, rule := range ingress.Spec.Rules { + for _, p := range rule.HTTP.Paths { + + cond, _ := buildCondition(rule.Host, p.Path, p.PathType, annotationConds) + + product := DefaultProduct + if rule.Host != "" { + product = rule.Host + } + + if _, ok := c.rules[product]; !ok { + return fmt.Errorf("rollback unknown product") + } + // Note: cause cond cannot repeated, so we do not need to judge refCount in routeRule + if _, ok := c.rules[product][cond]; ok { + delete(c.rules[product], cond) + } + } + } + + return nil +} + +func (c *BfeRouteConfigBuilder) Build() error { + clusterConf, err := c.buildBfeClusterConf() + if err != nil { + return err + } + hostConf := c.buildHostTableConf() + route, err := c.buildRouteTableConfFile() + if err != nil { + return err + } + c.routeConf = BfeRouteConf{ + hostTableConf: &hostConf, + routeTableFile: &route, + bfeClusterConf: &clusterConf, + } + return nil +} + +func (c *BfeRouteConfigBuilder) buildBfeClusterConf() (cluster_conf.BfeClusterConf, error) { + clusterToConf := make(cluster_conf.ClusterToConf) + + clusterConfs := cluster_conf.BfeClusterConf{ + Version: &c.version, + Config: &clusterToConf, + } + + for _, rules := range c.rules { + for _, rule := range rules { + clusterName := rule.rule.RouteRuleFile.ClusterName + gslbConf := InitClusterGslb() + clusterToConf[*clusterName] = cluster_conf.ClusterConf{GslbBasic: gslbConf} + } + } + + err := cluster_conf.ClusterToConfCheck(clusterToConf) + if err != nil { + return clusterConfs, err + } + + return clusterConfs, nil +} + +func (c *BfeRouteConfigBuilder) buildHostTableConf() host_rule_conf.HostTableConf { + hostTagToHost := make(host_rule_conf.HostTagToHost) + productToHostTag := make(host_rule_conf.ProductToHostTag) + + // build for default product + defaultProduct := DefaultProduct + defaultHostList := host_rule_conf.HostnameList{defaultProduct} + hostTagToHost[defaultProduct] = &defaultHostList + defaultProductList := host_rule_conf.HostTagList{defaultProduct} + productToHostTag[defaultProduct] = &defaultProductList + + // build for custom product + for host := range c.rules { + product := host + hostnameList := host_rule_conf.HostnameList{host} + hostTagToHost[product] = &hostnameList + list := host_rule_conf.HostTagList{product} + productToHostTag[product] = &list + } + + return host_rule_conf.HostTableConf{ + Version: &c.version, + DefaultProduct: &defaultProduct, + Hosts: &hostTagToHost, + HostTags: &productToHostTag, + } +} + +// buildRouteTableConfFile builds route table for all product from ingress rules +func (c *BfeRouteConfigBuilder) buildRouteTableConfFile() (route_rule_conf.RouteTableFile, error) { + var routeTable route_rule_conf.RouteTableFile + productBasicRouteRule := make(route_rule_conf.ProductBasicRouteRuleFile) + routeTable.BasicRule = &productBasicRouteRule + productAdvancedRouteRule := make(route_rule_conf.ProductAdvancedRouteRuleFile) + routeTable.ProductRule = &productAdvancedRouteRule + + cov := newAdvancedRuleCoverage() + for host, rules := range c.rules { + // collect rules + var routeRuleFiles []ingressRouteRuleFile + for _, rule := range rules { + routeRuleFiles = append(routeRuleFiles, *rule.rule) + } + + // sort rules + sortRules(routeRuleFiles) + + // build rules and save to routeTable + for _, routeRuleFile := range routeRuleFiles { + buildRouteRule(host, routeRuleFile, cov, routeTable) + } + } + routeTable.Version = &c.version + return routeTable, nil +} + +/* +buildRouteRule builds route rules, it's stateful. + +Route rule is built according to current host, rule file and coverage, +and append built rule to result RouteTableFile +*/ +func buildRouteRule(host string, ruleFile ingressRouteRuleFile, cov *advancedRuleCoverage, + result route_rule_conf.RouteTableFile) { + // basic route rule can't satisfied ingress with advanced BFE annotation + if len(ruleFile.RawRuleInfo.Annotations) <= 0 { + buildBasicRouteRule(host, ruleFile, cov, result) + } else { + // update advanced rule coverage + cov.Cover(ruleFile.RawRuleInfo) + + buildProductRouteRule(host, ruleFile, result) + } +} + +/* +buildBasicRouteRule builds basic route rules, it's stateful. + +Basic route rule is built according to current host, rule file and coverage, +and append built rule to result RouteTableFile. + +If current basic route rule is covered by previous product route rule, +it will convert to advanced mode, and corresponding new product route rule is appended. +*/ +func buildBasicRouteRule(host string, ruleFile ingressRouteRuleFile, cov *advancedRuleCoverage, + result route_rule_conf.RouteTableFile) { + basicRule := newBasicRouteRuleFile(ruleFile) + + if cov.IsCovered(ruleFile.RawRuleInfo) { + // convert to advanced mode if covered by any advanced rule + basicRule.ClusterName = pointer.StringPtr(ClusterNameAdvancedMode) + buildProductRouteRule(host, ruleFile, result) + } + + (*result.BasicRule)[host] = append((*result.BasicRule)[host], basicRule) +} + +/* +buildProductRouteRule builds advanced route rules, it's stateful. + +Product route rule is built according to current host and rule file, +and append built rule to result RouteTableFile +*/ +func buildProductRouteRule(host string, ruleFile ingressRouteRuleFile, result route_rule_conf.RouteTableFile) { + advancedRule := route_rule_conf.AdvancedRouteRuleFile{ + Cond: ruleFile.RouteRuleFile.Cond, + ClusterName: ruleFile.RouteRuleFile.ClusterName, + } + (*result.ProductRule)[host] = append((*result.ProductRule)[host], advancedRule) +} + +func (c *BfeRouteConfigBuilder) Dump() error { + err := c.dumper.DumpJson(c.routeConf.hostTableConf, HostRuleData) + if err != nil { + return fmt.Errorf("dump %s error: %v", HostRuleData, err) + } + + err = c.dumper.DumpJson(c.routeConf.routeTableFile, RouteRuleData) + if err != nil { + return fmt.Errorf("dump %s error: %v", RouteRuleData, err) + } + + err = c.dumper.DumpJson(c.routeConf.bfeClusterConf, ClusterConfData) + if err != nil { + return fmt.Errorf("dump %s error: %v", ClusterConfData, err) + } + + return nil +} + +func (c *BfeRouteConfigBuilder) Reload() error { + return c.reloader.DoReload(c.routeConf, ConfigNameRouteConf) +} + +func buildCondition(host string, path string, pathType *networking.PathType, exConds []BfeAnnotation) (string, int) { + condType := UnknownConditionType + bfePathType := BfePathType(pathType) + + stmts := make([]string, 0) + + hostStmt, hostType := hostStatement(host) + pathStmt := pathStatement(path, bfePathType) + stmts = append(stmts, hostStmt, pathStmt) + + // set condition type + switch hostType { + case HostTypeNoRestriction: + if len(path) == 0 { + return expression(stmts), ConditionTypeContainNoHostPath + } + + switch bfePathType { + case networking.PathTypeExact: + condType = ConditionTypeContainOnlyExactPath + default: + condType = ConditionTypeContainOnlyPrefixPath + } + + case HostTypeWildcard: + if len(path) == 0 { + condType = ConditionTypeContainOnlyWildcardHost + break + } + + switch bfePathType { + case networking.PathTypeExact: + condType = ConditionTypeContainWildcardHostExactPath + default: + condType = ConditionTypeContainWildcardHostPrefixPath + } + + case HostTypeExact: + if len(path) == 0 { + condType = ConditionTypeContainOnlyExactHost + break + } + + switch bfePathType { + case networking.PathTypeExact: + condType = ConditionTypeContainExactHostExactPath + default: + condType = ConditionTypeContainExactHostPrefixPath + } + } + + for _, exCond := range exConds { + stmts = append(stmts, exCond.Build()) + } + return expression(stmts), condType +} + +func sortRules(routeRuleFiles []ingressRouteRuleFile) { + sort.Slice(routeRuleFiles, func(i, j int) bool { + // Sort by ConditionType. + // As ContainHostPath > ContainOnlyHost > ContainOnlyPath > ContainNoHostPath + if routeRuleFiles[i].ConditionType != routeRuleFiles[j].ConditionType { + return routeRuleFiles[i].ConditionType < routeRuleFiles[j].ConditionType + } + + // Sort by Host length if ConditionType is same, more exact host with higher weight; + // as host(www.baidu.com) with higher weight than host(baidu.com) + if len(routeRuleFiles[i].RawRuleInfo.Host) != len(routeRuleFiles[j].RawRuleInfo.Host) { + return len(routeRuleFiles[i].RawRuleInfo.Host) > len(routeRuleFiles[j].RawRuleInfo.Host) + } + + // Sort by Path if Host length is same, more exact path with higher weight; + // as path(/api/v1/route) with higher weight than path(/api/v1) + if len(routeRuleFiles[i].RawRuleInfo.Path) != len(routeRuleFiles[j].RawRuleInfo.Path) { + return len(routeRuleFiles[i].RawRuleInfo.Path) > len(routeRuleFiles[j].RawRuleInfo.Path) + } + + // Sort by quantity of annotations if the path is same; + // as condition with header and cookie with higher weight than header; + if len(routeRuleFiles[i].RawRuleInfo.Annotations) != len(routeRuleFiles[j].RawRuleInfo.Annotations) { + return len(routeRuleFiles[i].RawRuleInfo.Annotations) > len(routeRuleFiles[j].RawRuleInfo.Annotations) + } + + // Sort by each annotation's Priority as quantity of annotations is same; + // for example, cookie is greater than header; + for index := 0; index < len(routeRuleFiles[i].RawRuleInfo.Annotations); index++ { + if routeRuleFiles[i].RawRuleInfo.Annotations[index].Priority() != routeRuleFiles[j].RawRuleInfo.Annotations[index].Priority() { + return routeRuleFiles[i].RawRuleInfo.Annotations[index].Priority() < routeRuleFiles[j].RawRuleInfo.Annotations[index].Priority() + } + } + + // Sort by length of condition if all above is same. + if len(*routeRuleFiles[i].RouteRuleFile.Cond) != len(*routeRuleFiles[j].RouteRuleFile.Cond) { + return len(*routeRuleFiles[i].RouteRuleFile.Cond) > len(*routeRuleFiles[j].RouteRuleFile.Cond) + } + + // Sort by content of condition if all above is same. + if *routeRuleFiles[i].RouteRuleFile.Cond != *routeRuleFiles[j].RouteRuleFile.Cond { + return *routeRuleFiles[i].RouteRuleFile.Cond > *routeRuleFiles[j].RouteRuleFile.Cond + } + + // Sort by cluster name if condition is same. + return *routeRuleFiles[i].RouteRuleFile.ClusterName > *routeRuleFiles[j].RouteRuleFile.ClusterName + }) +} + +func InitClusterGslb() *cluster_conf.GslbBasicConf { + gslbConf := &cluster_conf.GslbBasicConf{} + + defaultRetryMax := 2 + gslbConf.RetryMax = &defaultRetryMax + defaultCrossRetry := 0 + gslbConf.CrossRetry = &defaultCrossRetry + + defaultHashStrategy := cluster_conf.ClientIpOnly + defaultSessionSticky := false + gslbConf.HashConf = &cluster_conf.HashConf{ + HashStrategy: &defaultHashStrategy, + SessionSticky: &defaultSessionSticky, + } + + defaultBalMode := cluster_conf.BalanceModeWrr + gslbConf.BalanceMode = &defaultBalMode + return gslbConf +} + +func newBasicRouteRuleFile(rule ingressRouteRuleFile) route_rule_conf.BasicRouteRuleFile { + return route_rule_conf.BasicRouteRuleFile{ + Hostname: []string{rule.RawRuleInfo.Host}, + Path: []string{rule.RawRuleInfo.GetPathPattern()}, + ClusterName: rule.RouteRuleFile.ClusterName, + } +} + +// expression builds final expression from statements with AND logic +// empty statement is allowed, and it will be ignored; +// if no valuable statement is provided, return default_t() +func expression(stmts []string) string { + expressions := make([]string, 0) + for _, stmt := range stmts { + if len(stmt) > 0 { + expressions = append(expressions, stmt) + } + } + + if len(expressions) == 0 { + return "default_t()" + } + return strings.Join(expressions, " && ") +} + +// hostStatement builds host statement in condition, host type is judged by the way +func hostStatement(host string) (string, int) { + if len(host) == 0 { + return "", HostTypeNoRestriction + } + + if strings.HasPrefix(host, "*.") { + return fmt.Sprintf(`req_host_suffix_in("%s")`, host[1:]), HostTypeWildcard + } else { + return fmt.Sprintf(`req_host_in("%s")`, host), HostTypeExact + } +} + +// hostStatement builds path statement in condition +// see: https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types +func pathStatement(path string, bfePathType networking.PathType) string { + if len(path) == 0 { + return "" // no restriction + } + + if bfePathType == networking.PathTypeExact { + return fmt.Sprintf(`req_path_in("%s", false)`, path) + } else { + path = strings.TrimRight(path, "/") + if len(path) == 0 { + return "" // no restriction + } + return fmt.Sprintf(`(req_path_in("%s", false) || req_path_prefix_in("%s/", false))`, path, path) + } +} + +func (i *ingressRawRuleInfo) GetPathType() networking.PathType { + return BfePathType(i.PathType) +} + +// GetPathPattern return path pattern according to path type +// Return: +// prefix: {/path}/* +// exact: {/path} +func (i *ingressRawRuleInfo) GetPathPattern() string { + switch i.GetPathType() { + case networking.PathTypeExact: + return i.Path + default: + return path.Join(i.Path, "*") + } +} + +func newAdvancedRuleCoverage() *advancedRuleCoverage { + cov := new(advancedRuleCoverage) + cov.HostPath = make(map[string]map[string]bool) + return cov +} + +// Cover records host & path pattern covered by advanced rule +func (c *advancedRuleCoverage) Cover(advancedRule ingressRawRuleInfo) { + if _, ok := c.HostPath[advancedRule.Host]; !ok { + c.HostPath[advancedRule.Host] = make(map[string]bool) + } + c.HostPath[advancedRule.Host][advancedRule.GetPathPattern()] = true +} + +// IsCovered checks if a basic rule be overlapped with any known advanced rule +func (c *advancedRuleCoverage) IsCovered(basicRule ingressRawRuleInfo) bool { + if !strings.HasPrefix(basicRule.Host, "*.") { + // basic rule with exact host only overlapped with advanced rule with same exact host + if _, ok := c.HostPath[basicRule.Host]; !ok { + return false + } + return c.isPathOverlapped(basicRule, basicRule.Host) + } else { + // basic rule with wildcard host overlapped with any advanced rule with longer host (both exact & suffix) + basicRuleHost := basicRule.Host[1:] // "*.bar.foo" ==> ".bar.foo" + for advancedRuleHost := range c.HostPath { + if strings.HasSuffix(advancedRuleHost, basicRuleHost) && c.isPathOverlapped(basicRule, advancedRuleHost) { + return true + } + } + } + return false +} + +// isPathOverlapped check if a basic rule be overlapped with any known advanced rule with given host +func (c *advancedRuleCoverage) isPathOverlapped(basicRule ingressRawRuleInfo, advancedRuleHost string) bool { + switch basicRule.GetPathType() { + case networking.PathTypeExact: + // basic rule with exact path only overlapped with advanced rule with same exact path + return c.HostPath[advancedRuleHost][basicRule.Path] + default: + // basic rule with prefix path overlapped with any advanced rule with longer path (both exact & prefix) + for advancedRulePath := range c.HostPath[advancedRuleHost] { + if strings.HasPrefix(advancedRulePath, basicRule.Path) { + return true + } + } + return false + } +} diff --git a/internal/builder/route_builder_test.go b/internal/builder/route_builder_test.go new file mode 100644 index 00000000..468e26a4 --- /dev/null +++ b/internal/builder/route_builder_test.go @@ -0,0 +1,981 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builder + +import ( + "encoding/json" + "os" + "reflect" + "testing" +) + +import ( + "github.com/bfenetworks/bfe/bfe_config/bfe_route_conf/route_rule_conf" + "github.com/stretchr/testify/assert" + networking "k8s.io/api/networking/v1beta1" + "k8s.io/apimachinery/pkg/util/yaml" +) + +const ( + DemoHostExact = "example.host.com" + DemoHostWildcard = "*.example.host.com" +) + +func Test_buildCondition(t *testing.T) { + type args struct { + host string + path string + exConds []BfeAnnotation + } + tests := []struct { + name string + args args + want string + wantType int + }{ + { + name: "host + path", + args: args{ + host: "example.com", + path: "/foo", + exConds: nil, + }, + want: `req_host_in("example.com") && (req_path_in("/foo", false) || req_path_prefix_in("/foo/", false))`, + wantType: ConditionTypeContainExactHostPrefixPath, + }, + { + name: "only host", + args: args{ + host: "example.com", + path: "", + exConds: nil, + }, + want: "req_host_in(\"example.com\")", + wantType: ConditionTypeContainOnlyExactHost, + }, + { + name: "only path", + args: args{ + host: "", + path: "/foo", + exConds: nil, + }, + want: `(req_path_in("/foo", false) || req_path_prefix_in("/foo/", false))`, + wantType: ConditionTypeContainOnlyPrefixPath, + }, + { + name: "path end with /", + args: args{ + host: "", + path: "/foo/", + exConds: nil, + }, + want: `(req_path_in("/foo", false) || req_path_prefix_in("/foo/", false))`, + wantType: ConditionTypeContainOnlyPrefixPath, + }, + { + name: "no host and path", + args: args{ + host: "", + path: "", + exConds: nil, + }, + want: "default_t()", + wantType: ConditionTypeContainNoHostPath, + }, + { + name: "host + path + header", + args: args{ + host: "example.com", + path: "/foo", + exConds: []BfeAnnotation{ + &headerAnnotation{annotationStr: "HeaderTest: test"}, + }, + }, + want: `req_host_in("example.com") && (req_path_in("/foo", false) || req_path_prefix_in("/foo/", false)) && req_header_value_in("HeaderTest", "test", false)`, + wantType: ConditionTypeContainExactHostPrefixPath, + }, + { + name: "host + path + cookie", + args: args{ + host: "example.com", + path: "/foo", + exConds: []BfeAnnotation{ + &cookieAnnotation{annotationStr: "CookieTest: test"}, + }, + }, + want: `req_host_in("example.com") && (req_path_in("/foo", false) || req_path_prefix_in("/foo/", false)) && req_cookie_value_in("CookieTest", "test", false)`, + wantType: ConditionTypeContainExactHostPrefixPath, + }, + { + name: "no host and path, with cookie", + args: args{ + host: "", + path: "", + exConds: []BfeAnnotation{ + &cookieAnnotation{annotationStr: "CookieTest: test"}, + }, + }, + want: "default_t()", + wantType: ConditionTypeContainNoHostPath, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := buildCondition(tt.args.host, tt.args.path, nil, tt.args.exConds) + if got != tt.want { + t.Errorf("buildCondition() got = %v \n want %v", got, tt.want) + } + if got1 != tt.wantType { + t.Errorf("buildCondition() got1 = %v \n want %v", got1, tt.wantType) + } + }) + } +} + +func Test_sortRules(t *testing.T) { + var cluster = "test" + var condHostPathHeaderCookie = "req_host_in(\"example.com\") && req_path_prefix_in(\"/foo\", false) && req_cookie_value_in(\"CookieTest\", \"test\", false) && req_header_value_in(\"HeaderTest\", \"test\", false)" + var condHostPathCookie = "req_host_in(\"example.com\") && req_path_prefix_in(\"/foo\", false) && req_cookie_value_in(\"CookieTest\", \"test\", false)" + var condHostPathHeader = "req_host_in(\"example.com\") && req_path_prefix_in(\"/foo\", false) && req_header_value_in(\"HeaderTest\", \"test\", false)" + var condHostPath = "req_host_in(\"example.com\") && req_path_prefix_in(\"/foo\", false)" + var condHost = "req_host_in(\"example.com\")" + var condPath = "req_path_prefix_in(\"/foo\", false)" + var condHostCookie = "req_host_in(\"example.com\") && req_cookie_value_in(\"CookieTest\", \"test\", false)" + var condPathCookie = "req_host_in(\"example.com\") && req_cookie_value_in(\"CookieTest\", \"test\", false)" + var defaultStr = "default_t()" + + var ruleDefault = ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &defaultStr, + ClusterName: &cluster, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: "", + Path: "", + Annotations: []BfeAnnotation{ + &cookieAnnotation{annotationStr: "CookieTest: test"}, + &headerAnnotation{annotationStr: "HeaderTest: test"}, + }, + }, + ConditionType: ConditionTypeContainExactHostPrefixPath, + } + + var ruleHostPathHeaderCookie = ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &condHostPathHeaderCookie, + ClusterName: &cluster, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: "example.com", + Path: "/foo", + Annotations: []BfeAnnotation{ + &cookieAnnotation{annotationStr: "CookieTest: test"}, + &headerAnnotation{annotationStr: "HeaderTest: test"}, + }, + }, + ConditionType: ConditionTypeContainExactHostPrefixPath, + } + + var ruleHostPathCookie = ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &condHostPathCookie, + ClusterName: &cluster, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: "example.com", + Path: "/foo", + Annotations: []BfeAnnotation{ + &cookieAnnotation{annotationStr: "CookieTest: test"}, + }, + }, + ConditionType: ConditionTypeContainExactHostPrefixPath, + } + + var ruleHostPathHeader = ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &condHostPathHeader, + ClusterName: &cluster, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: "example.com", + Path: "/foo", + Annotations: []BfeAnnotation{ + &headerAnnotation{annotationStr: "HeaderTest: test"}, + }, + }, + ConditionType: ConditionTypeContainExactHostPrefixPath, + } + + var ruleHostPath = ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &condHostPath, + ClusterName: &cluster, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: "example.com", + Path: "/foo", + Annotations: nil, + }, + ConditionType: ConditionTypeContainExactHostPrefixPath, + } + + var ruleHost = ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &condHost, + ClusterName: &cluster, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: "example.com", + Path: "", + Annotations: nil, + }, + ConditionType: ConditionTypeContainOnlyExactHost, + } + + var rulePath = ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &condPath, + ClusterName: &cluster, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: "", + Path: "/foo", + Annotations: nil, + }, + ConditionType: ConditionTypeContainOnlyPrefixPath, + } + + var ruleHostCookie = ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &condHostCookie, + ClusterName: &cluster, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: "example.com", + Path: "", + Annotations: []BfeAnnotation{ + &cookieAnnotation{annotationStr: "CookieTest: test"}, + }, + }, + ConditionType: ConditionTypeContainOnlyExactHost, + } + var rulePathCookie = ingressRouteRuleFile{ + RouteRuleFile: route_rule_conf.AdvancedRouteRuleFile{ + Cond: &condPathCookie, + ClusterName: &cluster, + }, + RawRuleInfo: ingressRawRuleInfo{ + Host: "", + Path: "/foo", + Annotations: []BfeAnnotation{ + &cookieAnnotation{annotationStr: "CookieTest: test"}, + }, + }, + ConditionType: ConditionTypeContainOnlyPrefixPath, + } + + type args struct { + routeRuleFiles []ingressRouteRuleFile + } + tests := []struct { + name string + args args + want args + }{ + // TODO: Add test cases. + { + name: "", + args: args{ + routeRuleFiles: []ingressRouteRuleFile{ + ruleDefault, + ruleHost, + ruleHostCookie, + rulePath, + rulePathCookie, + ruleHostPath, + ruleHostPathCookie, + ruleHostPathHeader, + ruleHostPathHeaderCookie, + }, + }, + want: args{ + routeRuleFiles: []ingressRouteRuleFile{ + ruleHostPathHeaderCookie, + ruleHostPathCookie, + ruleHostPathHeader, + ruleHostPath, + ruleHostCookie, + ruleHost, + rulePathCookie, + rulePath, + ruleDefault, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sortRules(tt.args.routeRuleFiles) + if reflect.DeepEqual(tt.args.routeRuleFiles, tt.want.routeRuleFiles) { + t.Errorf("buildCondition() got1 = %v, want %v", tt.args.routeRuleFiles, tt.want.routeRuleFiles) + } + }) + } +} + +func TestBfeRouteConfigBuilder_Build(t *testing.T) { + testCases := map[string]interface{}{ + "single": map[string]interface{}{ + "annotation": map[string]interface{}{ + "cookie": BfeRouteConfigBuilderBuildCaseDefinedAnnotation, + "header": BfeRouteConfigBuilderBuildCaseDefinedAnnotation, + "load_balance": TestBfeRouteConfigBuilder_Build_CaseLoadBalance, + "other": TestBfeRouteConfigBuilder_Build_CaseOtherAnnotation, + }, + "host": map[string]map[string]func(t *testing.T){ + "basic": { + "wildcard": TestBfeRouteConfigBuilder_Build_CaseBasicRuleWildcardHost, + "exact": TestBfeRouteConfigBuilder_Build_CaseBasicRuleExactHost, + }, + "advanced": { + "wildcard": TestBfeRouteConfigBuilder_Build_CaseAdvancedRuleWildcardHost, + "exact": TestBfeRouteConfigBuilder_Build_CaseAdvancedRuleExactHost, + }, + }, + "path": map[string]func(t *testing.T, name string){ + "prefix": RouteConfigBuilderBuildCasePrefixPath, + "implementation_specific": RouteConfigBuilderBuildCasePrefixPath, + "non_path_type": RouteConfigBuilderBuildCasePrefixPath, + "exact": RouteConfigBuilderBuildCaseExactPath, + }, + }, + "multi": map[string]interface{}{ + "priority": map[string]map[string]func(t *testing.T){ + "path": { + "exact_path_basic_exact_path_advanced": TestBfeRouteConfigBuilder_Build_CasePriorityPath1, + "exact_path_basic_path_advanced": TestBfeRouteConfigBuilder_Build_CasePriorityPath2, + "exact_path_basic_prefix_path_advanced": TestBfeRouteConfigBuilder_Build_CasePriorityPath3, + "prefix_path_basic_exact_path_advanced": TestBfeRouteConfigBuilder_Build_CasePriorityPath4, + "prefix_path_basic_prefix_path_advanced": TestBfeRouteConfigBuilder_Build_CasePriorityPath5, + }, + }, + "conflict": TestBfeRouteConfigBuilder_Build_CaseConflict, + }, + } + + traverseTestCases(t, testCases) +} + +func TestBfeRouteConfigBuilder_Build2(t *testing.T) { + + b, err := routeConfigBuilderGenerator("normal") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + assert.Greater(t, len(*b.routeConf.routeTableFile.BasicRule), 0) + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.ProductRule)) + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) +} + +func BfeRouteConfigBuilderBuildCaseDefinedAnnotation(t *testing.T, name string) { + b, err := routeConfigBuilderGenerator("single/annotation/" + name) + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.BasicRule)) + assert.Greater(t, len(*b.routeConf.routeTableFile.ProductRule), 0) + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) +} + +func TestBfeRouteConfigBuilder_Build_CaseOtherAnnotation(t *testing.T) { + b, err := routeConfigBuilderGenerator("single/annotation/other") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + assert.Greater(t, len(*b.routeConf.routeTableFile.BasicRule), 0) + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.ProductRule)) + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) +} + +func RouteConfigBuilderBuildCasePrefixPath(t *testing.T, name string) { + b, err := routeConfigBuilderGenerator("single/path/" + name) + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + // path in basic rule with wildcard + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Greater(t, len(basicRules), 0) + demoProductRules := basicRules[DemoHostExact] + assert.Greater(t, len(demoProductRules), 0) + paths := demoProductRules[0].Path + assert.Greater(t, len(paths), 0) + assert.Equal(t, paths[0], "/foo/*") + // no advanced rule + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.ProductRule)) + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) +} + +func RouteConfigBuilderBuildCaseExactPath(t *testing.T, name string) { + b, err := routeConfigBuilderGenerator("single/path/" + name) + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + // host in basic rule + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Greater(t, len(basicRules), 0) + demoProductRules := basicRules[DemoHostExact] + assert.Greater(t, len(demoProductRules), 0) + paths := demoProductRules[0].Path + assert.Greater(t, len(paths), 0) + assert.Equal(t, paths[0], "/foo") + // no advanced rule + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.ProductRule)) + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) +} + +func TestBfeRouteConfigBuilder_Build_CaseBasicRuleWildcardHost(t *testing.T) { + b, err := routeConfigBuilderGenerator("single/host/basic/wildcard") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + // host in basic rule + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Greater(t, len(basicRules), 0) + demoProductRules := basicRules[DemoHostWildcard] + assert.Greater(t, len(demoProductRules), 0) + hosts := demoProductRules[0].Hostname + assert.Greater(t, len(hosts), 0) + assert.Equal(t, hosts[0], DemoHostWildcard) + // no advanced rule + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.ProductRule)) + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) +} + +func TestBfeRouteConfigBuilder_Build_CaseBasicRuleExactHost(t *testing.T) { + b, err := routeConfigBuilderGenerator("single/host/basic/exact") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + // host in basic rule + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Greater(t, len(basicRules), 0) + demoProductRules := basicRules[DemoHostExact] + assert.Greater(t, len(demoProductRules), 0) + hosts := demoProductRules[0].Hostname + assert.Greater(t, len(hosts), 0) + assert.Equal(t, hosts[0], DemoHostExact) + // no advanced rule + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.ProductRule)) + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) +} + +func TestBfeRouteConfigBuilder_Build_CaseAdvancedRuleWildcardHost(t *testing.T) { + b, err := routeConfigBuilderGenerator("single/host/advanced/wildcard") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + // no basic rule + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.BasicRule)) + // host in advanced rule + advancedRule := *b.routeConf.routeTableFile.ProductRule + assert.Greater(t, len(advancedRule), 0) + demoProductRules := advancedRule[DemoHostWildcard] + assert.Greater(t, len(demoProductRules), 0) + cond := demoProductRules[0].Cond + assert.NotNil(t, cond) + assert.Contains(t, *cond, `req_host_suffix_in(".example.host.com")`) + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) +} + +func TestBfeRouteConfigBuilder_Build_CaseAdvancedRuleExactHost(t *testing.T) { + b, err := routeConfigBuilderGenerator("single/host/advanced/exact") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + // no basic rule + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.BasicRule)) + // host in advanced rule + advancedRule := *b.routeConf.routeTableFile.ProductRule + assert.Greater(t, len(advancedRule), 0) + demoProductRules := advancedRule[DemoHostExact] + assert.Greater(t, len(demoProductRules), 0) + cond := demoProductRules[0].Cond + assert.NotNil(t, cond) + assert.Contains(t, *cond, `req_host_in("example.host.com")`) + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) +} + +func TestBfeRouteConfigBuilder_Build_CaseLoadBalance(t *testing.T) { + b, _ := routeConfigBuilderGenerator("single/annotation/load_balance") + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + assert.Greater(t, len(*b.routeConf.routeTableFile.BasicRule), 0) + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.ProductRule)) +} + +func TestBfeRouteConfigBuilder_Build_CaseConflict(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/conflict") + assert.NotNil(t, err, "expect ingress conflicts") + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Greater(t, len(basicRules), 0) + demoProductRules := basicRules["example.foo.com"] + assert.Greater(t, len(demoProductRules), 0) + // previous ingress rule + assert.Contains(t, *demoProductRules[0].ClusterName, "service1") + assert.Equal(t, 0, len(*b.routeConf.routeTableFile.ProductRule)) +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityPath1(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/path/exact_path_basic_exact_path_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 1, len(basicRules)) // product count + demoProductBasicRules := basicRules[DemoHostExact] + assert.Equal(t, 2, len(demoProductBasicRules)) // basic rule count + assert.Contains(t, *demoProductBasicRules[0].ClusterName, "service4") + assert.Contains(t, demoProductBasicRules[0].Path, "/foo") + assert.Equal(t, "ADVANCED_MODE", *demoProductBasicRules[1].ClusterName) + assert.Contains(t, demoProductBasicRules[1].Path, "/bar") + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 1, len(advancedRules)) // product count + demoProductAdvancedRules := advancedRules[DemoHostExact] + assert.Equal(t, 2, len(demoProductAdvancedRules)) // advanced rule count + assert.Contains(t, *demoProductAdvancedRules[0].ClusterName, "service1") + assert.Contains(t, *demoProductAdvancedRules[1].ClusterName, "service3") +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityPath2(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/path/exact_path_basic_prefix_path_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 1, len(basicRules)) // product count + demoProductBasicRules := basicRules[DemoHostExact] + assert.Equal(t, 1, len(demoProductBasicRules)) // basic rule count + assert.Contains(t, demoProductBasicRules[0].Path, "/foo/bar") + assert.Contains(t, *demoProductBasicRules[0].ClusterName, "service4") + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 1, len(advancedRules)) // product count + demoProductAdvancedRules := advancedRules[DemoHostExact] + assert.Equal(t, 3, len(demoProductAdvancedRules)) // advanced rule count + assert.Contains(t, *demoProductAdvancedRules[0].ClusterName, "service3") + assert.Contains(t, *demoProductAdvancedRules[1].ClusterName, "service2") + assert.Contains(t, *demoProductAdvancedRules[2].ClusterName, "service1") +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityPath3(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/path/exact_path_basic_path_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 1, len(basicRules)) // product count + demoProductBasicRules := basicRules[DemoHostExact] + assert.Equal(t, 1, len(demoProductBasicRules)) // basic rule count + assert.Contains(t, demoProductBasicRules[0].Path, "/foo") + assert.Equal(t, "ADVANCED_MODE", *demoProductBasicRules[0].ClusterName) + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 1, len(advancedRules)) // product count + demoProductAdvancedRules := advancedRules[DemoHostExact] + assert.Equal(t, 3, len(demoProductAdvancedRules)) // advanced rule count + assert.Contains(t, *demoProductAdvancedRules[0].ClusterName, "service1") + assert.Contains(t, *demoProductAdvancedRules[1].ClusterName, "service4") + assert.Contains(t, *demoProductAdvancedRules[2].ClusterName, "service2") +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityPath4(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/path/prefix_path_basic_exact_path_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 1, len(basicRules)) // product count + demoProductBasicRules := basicRules[DemoHostExact] + assert.Equal(t, 2, len(demoProductBasicRules)) // basic rule count + assert.Contains(t, demoProductBasicRules[0].Path, "/foo/*") + assert.Equal(t, "ADVANCED_MODE", *demoProductBasicRules[0].ClusterName) + assert.Contains(t, demoProductBasicRules[1].Path, "/bar/*") + assert.Equal(t, "ADVANCED_MODE", *demoProductBasicRules[1].ClusterName) + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 1, len(advancedRules)) // product count + demoProductAdvancedRules := advancedRules[DemoHostExact] + assert.Equal(t, 4, len(demoProductAdvancedRules)) // advanced rule count + assert.Contains(t, *demoProductAdvancedRules[0].ClusterName, "service2") + assert.Contains(t, *demoProductAdvancedRules[1].ClusterName, "service1") + assert.Contains(t, *demoProductAdvancedRules[2].ClusterName, "service4") + assert.Contains(t, *demoProductAdvancedRules[3].ClusterName, "service3") +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityPath5(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/path/prefix_path_basic_prefix_path_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 1, len(basicRules)) // product count + demoProductBasicRules := basicRules[DemoHostExact] + assert.Equal(t, 3, len(demoProductBasicRules)) // basic rule count + assert.Contains(t, demoProductBasicRules[0].Path, "/bar/baz/*") + assert.Contains(t, *demoProductBasicRules[0].ClusterName, "service5") + assert.Contains(t, demoProductBasicRules[1].Path, "/foo/*") + assert.Equal(t, "ADVANCED_MODE", *demoProductBasicRules[1].ClusterName) + assert.Contains(t, demoProductBasicRules[2].Path, "/bar/*") + assert.Equal(t, "ADVANCED_MODE", *demoProductBasicRules[2].ClusterName) + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 1, len(advancedRules)) // product count + demoProductAdvancedRules := advancedRules[DemoHostExact] + assert.Equal(t, 4, len(demoProductAdvancedRules)) // advanced rule count + assert.Contains(t, *demoProductAdvancedRules[0].ClusterName, "service1") + assert.Contains(t, *demoProductAdvancedRules[1].ClusterName, "service2") + assert.Contains(t, *demoProductAdvancedRules[2].ClusterName, "service3") + assert.Contains(t, *demoProductAdvancedRules[3].ClusterName, "service4") +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityHost1(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/host/exact_host_basic_exact_host_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 2, len(basicRules)) // product count + assert.Equal(t, "ADVANCED_MODE", *basicRules["foo.host.com"][0].ClusterName) + assert.Contains(t, *basicRules["bar.host.com"][0].ClusterName, "service3") + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 1, len(advancedRules)) // product count + assert.Equal(t, 2, len(advancedRules["foo.host.com"])) // advanced rule count + assert.Contains(t, *advancedRules["foo.host.com"][0].ClusterName, "service1") + assert.Contains(t, *advancedRules["foo.host.com"][1].ClusterName, "service2") +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityHost2(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/host/exact_host_basic_wildcard_host_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 1, len(basicRules)) // product count + assert.Equal(t, 1, len(basicRules["foo.host.com"])) // basic rule count + assert.Contains(t, *basicRules["foo.host.com"][0].ClusterName, "service4") + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 3, len(advancedRules)) // product count + assert.Contains(t, *advancedRules["*.bar.foo.host.com"][0].ClusterName, "service1") + assert.Contains(t, *advancedRules["*.foo.host.com"][0].ClusterName, "service2") + assert.Contains(t, *advancedRules["*.host.com"][0].ClusterName, "service3") +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityHost3(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/host/exact_host_basic_host_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 1, len(basicRules)) // product count + assert.Equal(t, "ADVANCED_MODE", *basicRules["foo.host.com"][0].ClusterName) + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 2, len(advancedRules)) // product count + assert.Contains(t, *advancedRules["foo.host.com"][0].ClusterName, "service1") + assert.Contains(t, *advancedRules["foo.host.com"][1].ClusterName, "service3") + assert.Contains(t, *advancedRules["*.host.com"][0].ClusterName, "service2") +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityHost4(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/host/wildcard_host_basic_exact_host_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 3, len(basicRules)) // product count + assert.Equal(t, "ADVANCED_MODE", *basicRules["*.foo.host.com"][0].ClusterName) + assert.Contains(t, *basicRules["*.bar.host.com"][0].ClusterName, "service5") + assert.Equal(t, "ADVANCED_MODE", *basicRules["*.baz.host.com"][0].ClusterName) + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 5, len(advancedRules)) // product count + assert.Contains(t, *advancedRules["bar.foo.host.com"][0].ClusterName, "service1") + assert.Contains(t, *advancedRules["*.foo.host.com"][0].ClusterName, "service4") + assert.Contains(t, *advancedRules["bar.host.com"][0].ClusterName, "service2") + assert.Contains(t, *advancedRules["foo.bar.baz.host.com"][0].ClusterName, "service3") + assert.Contains(t, *advancedRules["*.baz.host.com"][0].ClusterName, "service6") +} + +func TestBfeRouteConfigBuilder_Build_CasePriorityHost5(t *testing.T) { + b, err := routeConfigBuilderGenerator("multi/priority/host/wildcard_host_basic_wildcard_host_advanced") + assert.Nil(t, err, err) + + // invoke Build() + if err := b.Build(); err != nil { + t.Errorf("Build(): %s", err) + } + + // verify + t.Logf("routeTableFile: %s", jsonify(b.routeConf.routeTableFile)) + + // 1 basic rule with ADVANCED_MODE + basicRules := *b.routeConf.routeTableFile.BasicRule + assert.Equal(t, 3, len(basicRules)) // product count + assert.Equal(t, "ADVANCED_MODE", *basicRules["*.foo.host.com"][0].ClusterName) + assert.Equal(t, "ADVANCED_MODE", *basicRules["*.bar.host.com"][0].ClusterName) + assert.Contains(t, *basicRules["*.baz.host.com"][0].ClusterName, "service6") + + // 2 advanced rule + advancedRules := *b.routeConf.routeTableFile.ProductRule + assert.Equal(t, 4, len(advancedRules)) // product count + assert.Contains(t, *advancedRules["*.bar.foo.host.com"][0].ClusterName, "service1") + assert.Contains(t, *advancedRules["*.foo.host.com"][0].ClusterName, "service4") + assert.Contains(t, *advancedRules["*.bar.host.com"][0].ClusterName, "service2") + assert.Contains(t, *advancedRules["*.bar.host.com"][1].ClusterName, "service5") + assert.Contains(t, *advancedRules["*.host.com"][0].ClusterName, "service3") +} + +// routeConfigBuilderGenerator generate route config builder from file +// Params: +// name: file name prefix +// Returns: +// *BfeRouteConfigBuilder: builder generated by non-conflicting ingresses +// error: error for last conflict/wrong ingress +func routeConfigBuilderGenerator(name string) (*BfeRouteConfigBuilder, error) { + + // load ingress from file + ingresses := loadIngress(name) + + // submit ingress to builder + var submitErr error + builder := NewBfeRouteConfigBuilder(nil, "0", nil, nil) + for _, ingress := range ingresses { + err := builder.Submit(ingress) + if err != nil { + submitErr = err + } + } + return builder, submitErr +} + +func loadIngress(name string) []*networking.Ingress { + fi, err := os.Open("./testdata/" + name + ".yaml") + if err != nil { + return nil + } + + list := make([]*networking.Ingress, 0) + decoder := yaml.NewYAMLOrJSONDecoder(fi, 4096) + for { + ingress := networking.Ingress{} + err := decoder.Decode(&ingress) + if err != nil { + return list + } + list = append(list, &ingress) + } + return list +} + +func jsonify(object interface{}) string { + bytes, err := json.MarshalIndent(object, "", " ") + if err != nil { + return "marshal route table failed" + } + return string(bytes) +} + +func traverseTestCases(t *testing.T, testCases interface{}) { + v := reflect.ValueOf(testCases) + for _, name := range v.MapKeys() { + t.Run(name.String(), func(t *testing.T) { + traverseNamedTestCases(t, name.String(), v.MapIndex(name).Interface()) + }) + } +} + +func traverseNamedTestCases(t *testing.T, name string, testCases interface{}) { + v := reflect.ValueOf(testCases) + switch v.Kind() { + case reflect.Func: + switch v.Interface().(type) { + case func(t *testing.T): + v.Interface().(func(t *testing.T))(t) + t.Logf("func(t *testing.T)") + + case func(t *testing.T, name string): + v.Interface().(func(t *testing.T, name string))(t, name) + t.Logf("func(t *testing.T, name string) \n") + + default: + t.Errorf("unexpect func: %+v \n", v.Interface()) + } + + case reflect.Map: + for _, name := range v.MapKeys() { + t.Run(name.String(), func(t *testing.T) { + traverseNamedTestCases(t, name.String(), v.MapIndex(name).Interface()) + }) + } + + default: + t.Errorf("unexpect kind: %+v \n", v.Kind()) + } +} diff --git a/internal/builder/testdata/multi/conflict.yaml b/internal/builder/testdata/multi/conflict.yaml new file mode 100644 index 00000000..b0b11cd6 --- /dev/null +++ b/internal/builder/testdata/multi/conflict.yaml @@ -0,0 +1,30 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: ingress-A +spec: + rules: + - host: example.foo.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + serviceName: service1 + servicePort: 80 + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: ingress-B +spec: + rules: + - host: example.foo.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + serviceName: service2 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/host/exact_host_basic_exact_host_advanced.yaml b/internal/builder/testdata/multi/priority/host/exact_host_basic_exact_host_advanced.yaml new file mode 100644 index 00000000..59ecd7c0 --- /dev/null +++ b/internal/builder/testdata/multi/priority/host/exact_host_basic_exact_host_advanced.yaml @@ -0,0 +1,40 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: foo.host.com + http: + paths: + - path: /foo + backend: + serviceName: service1 + servicePort: 80 + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: foo.host.com + http: + paths: + - path: /foo + backend: + serviceName: service2 + servicePort: 80 + - host: bar.host.com + http: + paths: + - path: /foo + backend: + serviceName: service3 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/host/exact_host_basic_host_advanced.yaml b/internal/builder/testdata/multi/priority/host/exact_host_basic_host_advanced.yaml new file mode 100644 index 00000000..5c680e1d --- /dev/null +++ b/internal/builder/testdata/multi/priority/host/exact_host_basic_host_advanced.yaml @@ -0,0 +1,40 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: foo.host.com + http: + paths: + - path: /foo + backend: + serviceName: service1 + servicePort: 80 + - host: "*.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service2 + servicePort: 80 + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: foo.host.com + http: + paths: + - path: /foo + backend: + serviceName: service3 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/host/exact_host_basic_wildcard_host_advanced.yaml b/internal/builder/testdata/multi/priority/host/exact_host_basic_wildcard_host_advanced.yaml new file mode 100644 index 00000000..972c7329 --- /dev/null +++ b/internal/builder/testdata/multi/priority/host/exact_host_basic_wildcard_host_advanced.yaml @@ -0,0 +1,48 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: "*.bar.foo.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service1 + servicePort: 80 + - host: "*.foo.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service2 + servicePort: 80 + - host: "*.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service3 + servicePort: 80 + + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: foo.host.com + http: + paths: + - path: /foo + backend: + serviceName: service4 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/host/wildcard_host_basic_exact_host_advanced.yaml b/internal/builder/testdata/multi/priority/host/wildcard_host_basic_exact_host_advanced.yaml new file mode 100644 index 00000000..01205537 --- /dev/null +++ b/internal/builder/testdata/multi/priority/host/wildcard_host_basic_exact_host_advanced.yaml @@ -0,0 +1,61 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: bar.foo.host.com + http: + paths: + - path: /foo + backend: + serviceName: service1 + servicePort: 80 + - host: bar.host.com + http: + paths: + - path: /foo + backend: + serviceName: service2 + servicePort: 80 + - host: foo.bar.baz.host.com + http: + paths: + - path: /foo + backend: + serviceName: service3 + servicePort: 80 + +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: "*.foo.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service4 + servicePort: 80 + - host: "*.bar.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service5 + servicePort: 80 + - host: "*.baz.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service6 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/host/wildcard_host_basic_wildcard_host_advanced.yaml b/internal/builder/testdata/multi/priority/host/wildcard_host_basic_wildcard_host_advanced.yaml new file mode 100644 index 00000000..4d2e38dd --- /dev/null +++ b/internal/builder/testdata/multi/priority/host/wildcard_host_basic_wildcard_host_advanced.yaml @@ -0,0 +1,61 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: "*.bar.foo.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service1 + servicePort: 80 + - host: "*.bar.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service2 + servicePort: 80 + - host: "*.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service3 + servicePort: 80 + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: "*.foo.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service4 + servicePort: 80 + - host: "*.bar.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service5 + servicePort: 80 + - host: "*.baz.host.com" + http: + paths: + - path: /foo + backend: + serviceName: service6 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/path/exact_path_basic_exact_path_advanced.yaml b/internal/builder/testdata/multi/priority/path/exact_path_basic_exact_path_advanced.yaml new file mode 100644 index 00000000..6b1642aa --- /dev/null +++ b/internal/builder/testdata/multi/priority/path/exact_path_basic_exact_path_advanced.yaml @@ -0,0 +1,40 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: example.host.com + http: + paths: + - path: /bar + pathType: Exact + backend: + serviceName: service1 + servicePort: 80 + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: example.host.com + http: + paths: + - path: /bar + pathType: Exact + backend: + serviceName: service3 + servicePort: 80 + - path: /foo + pathType: Exact + backend: + serviceName: service4 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/path/exact_path_basic_path_advanced.yaml b/internal/builder/testdata/multi/priority/path/exact_path_basic_path_advanced.yaml new file mode 100644 index 00000000..3ec8567f --- /dev/null +++ b/internal/builder/testdata/multi/priority/path/exact_path_basic_path_advanced.yaml @@ -0,0 +1,40 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + pathType: Exact + backend: + serviceName: service1 + servicePort: 80 + - path: /foo + pathType: Prefix + backend: + serviceName: service2 + servicePort: 80 + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + pathType: Exact + backend: + serviceName: service4 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/path/exact_path_basic_prefix_path_advanced.yaml b/internal/builder/testdata/multi/priority/path/exact_path_basic_prefix_path_advanced.yaml new file mode 100644 index 00000000..df3e39e6 --- /dev/null +++ b/internal/builder/testdata/multi/priority/path/exact_path_basic_prefix_path_advanced.yaml @@ -0,0 +1,43 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + backend: + serviceName: service1 + servicePort: 80 + - path: /foo/bar + backend: + serviceName: service2 + servicePort: 80 + - path: /foo/bar/baz + backend: + serviceName: service3 + servicePort: 80 + + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo/bar + pathType: Exact + backend: + serviceName: service4 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/path/prefix_path_basic_exact_path_advanced.yaml b/internal/builder/testdata/multi/priority/path/prefix_path_basic_exact_path_advanced.yaml new file mode 100644 index 00000000..2e2e154b --- /dev/null +++ b/internal/builder/testdata/multi/priority/path/prefix_path_basic_exact_path_advanced.yaml @@ -0,0 +1,45 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: example.host.com + http: + paths: + - path: /bar + pathType: Exact + backend: + serviceName: service1 + servicePort: 80 + - path: /foo/bar + pathType: Exact + backend: + serviceName: service2 + servicePort: 80 + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: example.host.com + http: + paths: + - path: /bar + pathType: Prefix + backend: + serviceName: service3 + servicePort: 80 + - path: /foo + pathType: Prefix + backend: + serviceName: service4 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/multi/priority/path/prefix_path_basic_prefix_path_advanced.yaml b/internal/builder/testdata/multi/priority/path/prefix_path_basic_prefix_path_advanced.yaml new file mode 100644 index 00000000..e04e0be7 --- /dev/null +++ b/internal/builder/testdata/multi/priority/path/prefix_path_basic_prefix_path_advanced.yaml @@ -0,0 +1,50 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "advanced" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo/bar + pathType: Prefix + backend: + serviceName: service1 + servicePort: 80 + - path: /bar + pathType: Prefix + backend: + serviceName: service2 + servicePort: 80 + +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "basic" + namespace: production +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + serviceName: service3 + servicePort: 80 + - path: /bar + pathType: Prefix + backend: + serviceName: service4 + servicePort: 80 + - path: /bar/baz + pathType: Prefix + backend: + serviceName: service5 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/normal.yaml b/internal/builder/testdata/normal.yaml new file mode 100644 index 00000000..480d4874 --- /dev/null +++ b/internal/builder/testdata/normal.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: ingress-A +spec: + rules: + - host: example.foo.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + serviceName: service2 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/annotation/cookie.yaml b/internal/builder/testdata/single/annotation/cookie.yaml new file mode 100644 index 00000000..f1e890da --- /dev/null +++ b/internal/builder/testdata/single/annotation/cookie.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "foo" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.cookie: "key: value" + +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service1 + servicePort: 80 + - path: /foo + backend: + serviceName: service2 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/annotation/header.yaml b/internal/builder/testdata/single/annotation/header.yaml new file mode 100644 index 00000000..eaeb61e8 --- /dev/null +++ b/internal/builder/testdata/single/annotation/header.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "foo" + namespace: production + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" + +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service1 + servicePort: 80 + - path: /foo + backend: + serviceName: service2 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/annotation/load_balance.yaml b/internal/builder/testdata/single/annotation/load_balance.yaml new file mode 100644 index 00000000..c6322db8 --- /dev/null +++ b/internal/builder/testdata/single/annotation/load_balance.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: tls-example-ingress + annotations: + bfe.ingress.kubernetes.io/balance.weight: '{"service": {"service1":80, "service2":20}}' +spec: + tls: + - hosts: + - https-example.foo.com + secretName: testsecret-tls + rules: + - host: https-example.foo.com + http: + paths: + - path: / + pathType: Prefix + backend: + serviceName: service + servicePort: 80 diff --git a/internal/builder/testdata/single/annotation/other.yaml b/internal/builder/testdata/single/annotation/other.yaml new file mode 100644 index 00000000..3fd20662 --- /dev/null +++ b/internal/builder/testdata/single/annotation/other.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: "foo" + namespace: production + annotations: + non.exsit.key: "key: value" + bfe.ingress.kubernetes.io/router.non_exsit_key: "key: value" + +spec: + rules: + - host: example.net + http: + paths: + - path: /bar + backend: + serviceName: service1 + servicePort: 80 + - path: /foo + backend: + serviceName: service2 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/host/advanced/exact.yaml b/internal/builder/testdata/single/host/advanced/exact.yaml new file mode 100644 index 00000000..2b31653f --- /dev/null +++ b/internal/builder/testdata/single/host/advanced/exact.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: exact-host-demo + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + pathType: Exact + backend: + serviceName: service1 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/host/advanced/wildcard.yaml b/internal/builder/testdata/single/host/advanced/wildcard.yaml new file mode 100644 index 00000000..c6121875 --- /dev/null +++ b/internal/builder/testdata/single/host/advanced/wildcard.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: wildcard-host-demo + annotations: + bfe.ingress.kubernetes.io/router.header: "key: value" +spec: + rules: + - host: '*.example.host.com' + http: + paths: + - path: /foo + pathType: Exact + backend: + serviceName: service1 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/host/basic/exact.yaml b/internal/builder/testdata/single/host/basic/exact.yaml new file mode 100644 index 00000000..5deaae14 --- /dev/null +++ b/internal/builder/testdata/single/host/basic/exact.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: exact-host-demo +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + pathType: Exact + backend: + serviceName: service1 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/host/basic/wildcard.yaml b/internal/builder/testdata/single/host/basic/wildcard.yaml new file mode 100644 index 00000000..4ab64ff1 --- /dev/null +++ b/internal/builder/testdata/single/host/basic/wildcard.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: wildcard-host-demo +spec: + rules: + - host: '*.example.host.com' + http: + paths: + - path: /foo + pathType: Exact + backend: + serviceName: service1 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/path/exact.yaml b/internal/builder/testdata/single/path/exact.yaml new file mode 100644 index 00000000..91b4e1d5 --- /dev/null +++ b/internal/builder/testdata/single/path/exact.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: exact-path-demo +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + pathType: Exact + backend: + serviceName: service1 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/path/implementation_specific.yaml b/internal/builder/testdata/single/path/implementation_specific.yaml new file mode 100644 index 00000000..ce0873c1 --- /dev/null +++ b/internal/builder/testdata/single/path/implementation_specific.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: implementation-specific-path-demo +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + pathType: ImplementationSpecific + backend: + serviceName: service1 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/path/non_path_type.yaml b/internal/builder/testdata/single/path/non_path_type.yaml new file mode 100644 index 00000000..9004a081 --- /dev/null +++ b/internal/builder/testdata/single/path/non_path_type.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: non-path-type-demo +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + backend: + serviceName: service1 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/testdata/single/path/prefix.yaml b/internal/builder/testdata/single/path/prefix.yaml new file mode 100644 index 00000000..c61eda1b --- /dev/null +++ b/internal/builder/testdata/single/path/prefix.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: prefix-path-demo +spec: + rules: + - host: example.host.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + serviceName: service1 + servicePort: 80 \ No newline at end of file diff --git a/internal/builder/tls_config_builder.go b/internal/builder/tls_config_builder.go new file mode 100644 index 00000000..e52c99d9 --- /dev/null +++ b/internal/builder/tls_config_builder.go @@ -0,0 +1,287 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package builder + +import ( + "fmt" + "reflect" +) + +import ( + "github.com/bfenetworks/bfe/bfe_config/bfe_tls_conf/server_cert_conf" + "github.com/bfenetworks/bfe/bfe_config/bfe_tls_conf/tls_rule_conf" + networking "k8s.io/api/networking/v1beta1" +) + +import ( + "github.com/bfenetworks/ingress-bfe/internal/kubernetes_client" +) + +const ( + DefaultCNName = "example.org" + DefaultCrtPath = "tls_conf/certs/example.crt" + DefaultCrtKey = "tls_conf/certs/example.key" +) + +type certKeyConf struct { + cert []byte + key []byte +} + +type BfeTLSConf struct { + serverCertConf server_cert_conf.BfeServerCertConf + certKeyConf map[string]certKeyConf + tlsRuleConf tls_rule_conf.BfeTlsRuleConf +} + +var ( + ServerCertData = "tls_conf/server_cert_conf.data" + TLSRuleData = "tls_conf/tls_rule_conf.data" + CertKeyFilePath = "tls_conf/certs/" + ConfigNameTLSConf = "tls_conf" + + SecretCrt = "tls.crt" + SecretKey = "tls.key" +) + +type BfeTLSConfigBuilder struct { + client *kubernetes_client.KubernetesClient + + dumper *Dumper + reloader *Reloader + + version string + + serverCertConf server_cert_conf.BfeServerCertConf + + certKeyConf map[string]certKeyConf + hostRefCount map[string]int + + tc *BfeTLSConf +} + +func NewBfeTLSConfigBuilder(client *kubernetes_client.KubernetesClient, version string, dumper *Dumper, + reloader *Reloader) *BfeTLSConfigBuilder { + c := &BfeTLSConfigBuilder{ + client: client, + dumper: dumper, + reloader: reloader, + version: version, + certKeyConf: make(map[string]certKeyConf), + hostRefCount: make(map[string]int), + } + return c +} + +func (c *BfeTLSConfigBuilder) CheckTLS(crt, pkey []byte, host string) bool { + // TODO: add pem check in this function + return true +} + +func (c *BfeTLSConfigBuilder) CheckTlsConflict(certKeyConf map[string]certKeyConf) bool { + for host, certKey := range certKeyConf { + if _, ok := c.certKeyConf[host]; ok { + if !reflect.DeepEqual(c.certKeyConf[host], certKey) { + return false + } + } + } + return true +} + +func (c *BfeTLSConfigBuilder) submitCertKeyMap(certKeyMap map[string]certKeyConf) error { + if !c.CheckTlsConflict(certKeyMap) { + var keys []string + for key := range certKeyMap { + keys = append(keys, key) + } + return fmt.Errorf("cert conflict in host %v", keys) + } + for host, cert := range certKeyMap { + c.certKeyConf[host] = cert + if _, ok := c.hostRefCount[host]; !ok { + c.hostRefCount[host] = 0 + } + c.hostRefCount[host]++ + } + return nil +} + +func (c *BfeTLSConfigBuilder) getCertKeyMap(ingress *networking.Ingress) (map[string]certKeyConf, error) { + certKeyMap := make(map[string]certKeyConf) + namespace := ingress.Namespace + for _, tlsRule := range ingress.Spec.TLS { + secretName := tlsRule.SecretName + secrets, err := c.client.GetSecretsByName(namespace, secretName) + if err != nil { + return nil, fmt.Errorf("submit ingress %s fail, get secrets err: %s", ingress.Name, err.Error()) + } + if _, exists := secrets.Data[SecretKey]; !exists { + return nil, fmt.Errorf("submit ingress %s tls error: %s secret has no %s", ingress.Name, secretName, SecretKey) + } + if _, exists := secrets.Data[SecretCrt]; !exists { + return nil, fmt.Errorf("submit ingress %s tls error: %s secret has no %s", ingress.Name, secretName, SecretCrt) + } + var crt = secrets.Data[SecretCrt] + var key = secrets.Data[SecretKey] + + Hosts := tlsRule.Hosts + for _, host := range Hosts { + if !c.CheckTLS(crt, key, host) { + return nil, fmt.Errorf("submit ingress tls error: check %s for host %s crt/key error ", secretName, host) + } + certKeyMap[host] = certKeyConf{ + cert: crt, + key: key, + } + } + } + return certKeyMap, nil +} + +func (c *BfeTLSConfigBuilder) Submit(ingress *networking.Ingress) error { + certKeyMap, err := c.getCertKeyMap(ingress) + if err != nil { + return err + } + return c.submitCertKeyMap(certKeyMap) +} + +func (c *BfeTLSConfigBuilder) Rollback(ingress *networking.Ingress) error { + for _, tlsRule := range ingress.Spec.TLS { + Hosts := tlsRule.Hosts + for _, host := range Hosts { + if _, ok := c.hostRefCount[host]; ok { + c.hostRefCount[host]-- + if c.hostRefCount[host] <= 0 { + delete(c.hostRefCount, host) + delete(c.certKeyConf, host) + } + } + } + } + return nil +} + +func (c *BfeTLSConfigBuilder) Build() error { + if len(c.certKeyConf) == 0 { + return c.buildDefault() + } + return c.buildCustom() +} + +func (c *BfeTLSConfigBuilder) buildDefault() error { + c.tc = &BfeTLSConf{ + serverCertConf: server_cert_conf.BfeServerCertConf{ + Version: c.version, + Config: server_cert_conf.ServerCertConfMap{ + Default: DefaultCNName, + CertConf: map[string]server_cert_conf.ServerCertConf{ + DefaultCNName: { + ServerCertFile: DefaultCrtPath, + ServerKeyFile: DefaultCrtKey, + }, + }, + }, + }, + } + c.buildTLSConfig() + return nil +} + +func (c *BfeTLSConfigBuilder) buildTLSConfig() error { + conf := make(map[string]*tls_rule_conf.TlsRuleConf) + for host := range c.certKeyConf { + conf[host] = &tls_rule_conf.TlsRuleConf{ + SniConf: []string{host}, + CertName: host, + } + } + + c.tc.tlsRuleConf = tls_rule_conf.BfeTlsRuleConf{ + Version: c.version, + Config: conf, + DefaultChacha20: false, + DefaultDynamicRecord: false, + DefaultNextProtos: []string{"http/1.1"}, + } + return nil +} + +func (c *BfeTLSConfigBuilder) buildCustom() error { + serverCertConfig := server_cert_conf.ServerCertConfMap{ + CertConf: make(map[string]server_cert_conf.ServerCertConf), + Default: "", + } + for host := range c.certKeyConf { + if serverCertConfig.Default == "" || serverCertConfig.Default > host { + serverCertConfig.Default = host + } + serverCertConfig.CertConf[host] = server_cert_conf.ServerCertConf{ + ServerCertFile: c.getCertFilePath(host), + ServerKeyFile: c.getKeyFilePath(host), + } + } + + c.tc = &BfeTLSConf{ + certKeyConf: c.certKeyConf, + serverCertConf: server_cert_conf.BfeServerCertConf{ + Version: c.version, + Config: serverCertConfig, + }, + } + c.buildTLSConfig() + return nil +} + +func (c *BfeTLSConfigBuilder) Dump() error { + // dump key and cert for hosts + for host, ck := range c.tc.certKeyConf { + certFile := c.getCertFilePath(host) + if err := c.dumper.DumpBytes(ck.cert, certFile); err != nil { + return fmt.Errorf("write [%s] cert file fail, err: %s", host, err) + } + + keyFile := c.getKeyFilePath(host) + if err := c.dumper.DumpBytes(ck.key, keyFile); err != nil { + return fmt.Errorf("write [%s] key file fail, err: %s", host, err) + } + } + + // dump server cert config + err := c.dumper.DumpJson(c.tc.serverCertConf, ServerCertData) + if err != nil { + return fmt.Errorf("dump server_cert_conf: %v", err) + } + + // dump TLS rule config + err = c.dumper.DumpJson(c.tc.tlsRuleConf, TLSRuleData) + if err != nil { + return fmt.Errorf("dump tls_rule_conf: %v", err) + } + + return nil +} + +func (c *BfeTLSConfigBuilder) Reload() error { + return c.reloader.DoReload(c.tc, ConfigNameTLSConf) +} + +func (c *BfeTLSConfigBuilder) getCertFilePath(host string) string { + return c.dumper.Join(CertKeyFilePath, host+".cer") +} + +func (c *BfeTLSConfigBuilder) getKeyFilePath(host string) string { + return c.dumper.Join(CertKeyFilePath, host+".key") +} diff --git a/internal/kubernetes_client/kubernetes_client.go b/internal/kubernetes_client/kubernetes_client.go new file mode 100644 index 00000000..ca6ad35e --- /dev/null +++ b/internal/kubernetes_client/kubernetes_client.go @@ -0,0 +1,367 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes_client + +import ( + "context" + "fmt" + "reflect" + "strings" + "time" +) + +import ( + "github.com/baidu/go-lib/log" + core "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + networking "k8s.io/api/networking/v1beta1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + util "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/informers" + v1 "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/retry" +) + +const ( + AllIngressClass = "" + IngressClassAnnotationKey = "kubernetes.io/ingress.class" +) + +type resourceEventHandler struct { + ev chan<- interface{} +} + +func (h *resourceEventHandler) OnAdd(obj interface{}) { + eventHandlerFunc(h.ev, obj, "add") +} + +func (h *resourceEventHandler) OnUpdate(oldObj, newObj interface{}) { + if reflect.DeepEqual(oldObj, newObj) { + return + } + log.Logger.Debug("oldObj{%v} newObj{%v}", oldObj, newObj) + eventHandlerFunc(h.ev, newObj, "update") +} + +func (h *resourceEventHandler) OnDelete(obj interface{}) { + eventHandlerFunc(h.ev, obj, "del") +} + +func eventHandlerFunc(events chan<- interface{}, obj interface{}, action string) { + select { + case events <- obj: + default: + } +} + +func newResourceEventHandler(events chan<- interface{}) cache.ResourceEventHandler { + return &cache.FilteringResourceEventHandler{ + FilterFunc: func(obj interface{}) bool { + return true + }, + Handler: &resourceEventHandler{ev: events}, + } +} + +type KubernetesCluster struct { + SupportIngressClass bool + SupportNetworking bool +} + +type KubernetesClient struct { + namespaces []string + watchAll bool + + watchLabel bool + labels []string + + watchedIngressClass string + + clientset *kubernetes.Clientset + factories map[string]informers.SharedInformerFactory + eventCh chan interface{} + stopCh chan struct{} + + cluster *KubernetesCluster +} + +func NewKubernetesClient() (*KubernetesClient, error) { + c := new(KubernetesClient) + c.factories = make(map[string]informers.SharedInformerFactory) + c.eventCh = make(chan interface{}, 1) + c.stopCh = make(chan struct{}) + + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("InClusterConfig error: %v", err) + } + c.clientset, err = kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("NewForConfig error: %v", err) + } + + c.cluster = &KubernetesCluster{ + SupportIngressClass: c.SupportIngressClassVersion(), + SupportNetworking: c.SupportIngressVersion(), + } + return c, nil +} + +func (c *KubernetesClient) lookupNamespace(ns string) string { + if c.watchAll { + return meta.NamespaceAll + } + return ns +} + +func (c *KubernetesClient) Watch(namespaces []string, labels []string, ingressClass string, ResyncPeriod time.Duration) <-chan interface{} { + c.namespaces = namespaces + if len(namespaces) == 0 { + c.namespaces = []string{meta.NamespaceAll} + c.watchAll = true + } + + if c.watchAll && len(labels) > 0 { + c.watchLabel = true + c.labels = labels + } + c.watchedIngressClass = ingressClass + + eventHandler := newResourceEventHandler(c.eventCh) + for _, ns := range c.namespaces { + factory := informers.NewSharedInformerFactoryWithOptions(c.clientset, ResyncPeriod, informers.WithNamespace(ns)) + if c.cluster.SupportNetworking { + factory.Networking().V1beta1().Ingresses().Informer().AddEventHandler(eventHandler) + } else { + factory.Extensions().V1beta1().Ingresses().Informer().AddEventHandler(eventHandler) + } + if c.cluster.SupportIngressClass { + factory.Networking().V1beta1().IngressClasses().Informer().AddEventHandler(eventHandler) + } + + resources := factory.Core().V1() + resources.Services().Informer().AddEventHandler(eventHandler) + resources.Endpoints().Informer().AddEventHandler(eventHandler) + resources.Secrets().Informer().AddEventHandler(eventHandler) + if ns == meta.NamespaceAll { + resources.Namespaces().Informer().AddEventHandler(eventHandler) + } + + go factory.Start(c.stopCh) + c.factories[ns] = factory + } + + return c.eventCh +} + +func (c *KubernetesClient) Close() { + close(c.stopCh) +} + +func (c *KubernetesClient) GetResources(namespace string) v1.Interface { + return c.factories[c.lookupNamespace(namespace)].Core().V1() +} + +func (c *KubernetesClient) GetEndpoints(namespace, name string) (*core.Endpoints, error) { + endpoint, err := c.GetResources(namespace).Endpoints().Lister().Endpoints(namespace).Get(name) + return endpoint, err +} + +func (c *KubernetesClient) GetService(namespace, name string) (*core.Service, error) { + service, err := c.GetResources(namespace).Services().Lister().Services(namespace).Get(name) + return service, err +} + +func (c *KubernetesClient) GetNamespaceByLabel() []*core.Namespace { + if !c.watchAll || !c.watchLabel { + return nil + } + labelsMap := make(map[string]string) + for _, label := range c.labels { + kV := strings.Split(label, "=") + if len(kV) != 2 { + continue + } + labelsMap[kV[0]] = kV[1] + } + labelSelector := labels.Set(labelsMap).AsSelector() + namespaces, err := c.GetResources(meta.NamespaceAll).Namespaces().Lister().List(labelSelector) + if err != nil { + log.Logger.Warn("fail to list namespace by label %s: %s", c.labels, err) + return nil + } + return namespaces +} + +func (c *KubernetesClient) GetSecretsByName(namespace, name string) (*core.Secret, error) { + secret, err := c.GetResources(namespace).Secrets().Lister().Secrets(namespace).Get(name) + return secret, err +} + +func (c *KubernetesClient) GetIngresses() []*networking.Ingress { + var result []*networking.Ingress + + for ns, factory := range c.factories { + ings, err := c.getAllIngresses(factory) + if err != nil { + log.Logger.Info("Failed to list ingresses in namespace %s: %s", ns, err) + continue + } + + if c.watchLabel { + targetNamespaces := c.GetNamespaceByLabel() + filterFunc := func(ing *networking.Ingress) bool { + for _, targetNs := range targetNamespaces { + if ing.Namespace == targetNs.GetName() { + return true + } + } + log.Logger.Debug("ns[%s] ingress[%s] filter by namespace", ing.Namespace, ing.Name) + return false + } + ings = c.filterIngress(ings, filterFunc) + } + result = append(result, ings...) + } + return c.filterIngress(result, c.filterIngressByClass) +} + +// getAllIngresses gets all ingresses from certain informer factory +func (c *KubernetesClient) getAllIngresses(factory informers.SharedInformerFactory) ([]*networking.Ingress, error) { + if !c.cluster.SupportNetworking { + extendsIngs, err := factory.Extensions().V1beta1().Ingresses().Lister().List(labels.Everything()) + if err != nil { + return nil, err + } + + ings := make([]*networking.Ingress, 0) + for _, ing := range extendsIngs { + netIng, err := c.convertFromExtensions(ing) + if err != nil { + continue + } + ings = append(ings, netIng) + } + return ings, nil + } + + return factory.Networking().V1beta1().Ingresses().Lister().List(labels.Everything()) +} + +func (c *KubernetesClient) convertFromExtensions(old *extensions.Ingress) (*networking.Ingress, error) { + data, err := old.Marshal() + if err != nil { + return nil, err + } + ni := &networking.Ingress{} + err = ni.Unmarshal(data) + if err != nil { + return nil, err + } + return ni, nil +} + +func (c *KubernetesClient) filterIngressByClass(ingress *networking.Ingress) bool { + if c.watchedIngressClass == AllIngressClass { + return true + } + if val, ok := ingress.Annotations[IngressClassAnnotationKey]; ok && val == c.watchedIngressClass { + return true + } + + if c.cluster.SupportIngressClass && ingress.Spec.IngressClassName != nil { + ns := c.lookupNamespace(ingress.Namespace) + ic, err := c.factories[ns].Networking().V1beta1().IngressClasses().Lister().Get(*ingress.Spec.IngressClassName) + + if err != nil || ic == nil { + return false + } + return true + } + log.Logger.Debug("ns[%s] ingress[%s] filter by ingress.class[%s]", ingress.Namespace, + ingress.Name, c.watchedIngressClass) + return false +} + +func (c *KubernetesClient) GetVersion() (*version.Info, error) { + return c.clientset.Discovery().ServerVersion() +} + +func (c *KubernetesClient) SupportIngressClassVersion() bool { + serverVersion, err := c.GetVersion() + log.Logger.Info("get server running version %v", serverVersion) + if err != nil { + return false + } + v118, _ := util.ParseGeneric("v1.18.0") + runningVersion, err := util.ParseGeneric(serverVersion.String()) + if err != nil { + return false + } + return runningVersion.AtLeast(v118) +} + +func (c *KubernetesClient) SupportIngressVersion() bool { + serverVersion, err := c.GetVersion() + log.Logger.Info("get server running version %v", serverVersion) + if err != nil { + return false + } + v114, _ := util.ParseGeneric("v1.14.0") + runningVersion, err := util.ParseGeneric(serverVersion.String()) + if err != nil { + return false + } + return runningVersion.AtLeast(v114) +} + +func (c *KubernetesClient) UpdateIngressAnnotation(namespace, name, annotation, msg string) error { + retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + ingressClient := c.clientset.ExtensionsV1beta1().Ingresses(namespace) + result, getErr := ingressClient.Get(context.TODO(), name, meta.GetOptions{}) + if getErr != nil { + return fmt.Errorf("Failed to get latest version of Ingress: %v", getErr) + } + if result.Annotations == nil { + result.Annotations = make(map[string]string) + } + if val, exists := result.Annotations[annotation]; exists { + if val == msg { + return nil + } + } + result.Annotations[annotation] = msg + _, updateErr := ingressClient.Update(context.TODO(), result, meta.UpdateOptions{}) + return updateErr + }) + return retryErr +} + +func (c *KubernetesClient) filterIngress(ingresses []*networking.Ingress, filter IngressFilterFunc) []*networking.Ingress { + result := make([]*networking.Ingress, 0) + for _, ing := range ingresses { + if ing != nil && filter(ing) { + result = append(result, ing) + } + } + return result +} + +type IngressFilterFunc func(*networking.Ingress) bool diff --git a/internal/utils/commons.go b/internal/utils/commons.go new file mode 100644 index 00000000..ed7bc5cc --- /dev/null +++ b/internal/utils/commons.go @@ -0,0 +1,41 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "strings" + "time" +) + +// Namespaces implements flag.Value interface as []string +type Namespaces []string + +// String implements flag.Value.String() +func (n *Namespaces) String() string { + return strings.Join(*n, ",") +} + +// Set implements flag.Value.Set() +func (n *Namespaces) Set(v string) error { + *n = append(*n, v) + return nil +} + +// ingress controller default settings +const ( + DefaultBfeConfigRoot = "/home/work/bfe/conf/" + DefaultReloadURLPrefix = "http://localhost:8421/reload/" + DefaultSyncPeriod = 20 * time.Second +)