From 93a90693603cf2dd64b14ca04016bb13fd200b32 Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Thu, 24 Aug 2023 15:07:46 -0400 Subject: [PATCH 1/6] feat: add grpc health, deprecate old health * adds a standard gRPC health check * moves HTTP health checks off the metrics port, to the app port * deprecates old health checks on metrics port Signed-off-by: Todd Baert --- Makefile | 4 +++- config/deployments/flagd/deployment.yaml | 4 ++-- .../flag-evaluation/connect_service.go | 20 +++++++++++++++++-- .../high_level_architecture.md | 13 ++++++++---- flagd/go.mod | 2 ++ flagd/go.sum | 4 ++++ 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index fce3ea1b7..c151ee9b2 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ ZD_TEST_NAMESPACE ?= flagd-zd-test ZD_CLIENT_IMG ?= zd-client:latest FLAGD_PROXY_IMG ?= flagd-proxy:latest FLAGD_PROXY_IMG_ZD ?= flagd-proxy:zd +# the same "status.proto" is supplied by 2 modules, which causes an error we can safely ignore +export GOLANG_PROTOBUF_REGISTRATION_CONFLICT=warn workspace-init: workspace-clean go work init @@ -35,7 +37,7 @@ docker-push-flagd: build: workspace-init # default to flagd make build-flagd build-flagd: - go build -ldflags "-X main.version=dev -X main.commit=$$(git rev-parse --short HEAD) -X main.date=$$(date +%FT%TZ)" -o ./bin/flagd ./flagd + go build -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -X main.version=dev -X main.commit=$$(git rev-parse --short HEAD) -X main.date=$$(date +%FT%TZ)" -o ./bin/flagd ./flagd .PHONY: test test: # default to core make test-core diff --git a/config/deployments/flagd/deployment.yaml b/config/deployments/flagd/deployment.yaml index 0f7207430..d9d0e3e2d 100644 --- a/config/deployments/flagd/deployment.yaml +++ b/config/deployments/flagd/deployment.yaml @@ -29,13 +29,13 @@ spec: readinessProbe: httpGet: path: /readyz - port: 8014 + port: 8013 initialDelaySeconds: 5 periodSeconds: 5 livenessProbe: httpGet: path: /healthz - port: 8014 + port: 8013 initialDelaySeconds: 5 periodSeconds: 60 ports: diff --git a/core/pkg/service/flag-evaluation/connect_service.go b/core/pkg/service/flag-evaluation/connect_service.go index ddda9cd15..5528fb227 100644 --- a/core/pkg/service/flag-evaluation/connect_service.go +++ b/core/pkg/service/flag-evaluation/connect_service.go @@ -11,6 +11,7 @@ import ( "time" schemaConnectV1 "buf.build/gen/go/open-feature/flagd/bufbuild/connect-go/schema/v1/schemav1connect" + "connectrpc.com/grpchealth" "github.com/open-feature/flagd/core/pkg/eval" "github.com/open-feature/flagd/core/pkg/logger" "github.com/open-feature/flagd/core/pkg/service" @@ -101,6 +102,7 @@ func (s *ConnectService) Notify(n service.Notification) { s.eventingConfiguration.emitToAll(n) } +// nolint: funlen func (s *ConnectService) setupServer(svcConf service.Configuration) (net.Listener, error) { var lis net.Listener var err error @@ -129,16 +131,28 @@ func (s *ConnectService) setupServer(svcConf service.Configuration) (net.Listene path, handler := schemaConnectV1.NewServiceHandler(fes, append(svcConf.Options, marshalOpts)...) mux.Handle(path, handler) + mux.Handle(grpchealth.NewHandler(grpchealth.NewStaticChecker( + schemaConnectV1.ServiceName, + ))) + mux.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + mux.Handle("/readyz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.readinessEnabled && svcConf.ReadinessProbe() { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusPreconditionFailed) + } + })) s.serverMtx.Lock() s.server = &http.Server{ ReadHeaderTimeout: time.Second, - Handler: handler, + Handler: mux, } s.serverMtx.Unlock() // Add middlewares - metricsMiddleware := metricsmw.NewHTTPMetric(metricsmw.Config{ Service: svcConf.ServiceName, MetricRecorder: s.metrics, @@ -206,8 +220,10 @@ func (s *ConnectService) startMetricsServer(svcConf service.Configuration) error s.metricsServer.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/healthz": + s.logger.Warn(fmt.Sprintf("call to deprecated /healthz endpoint on port %d, use port %d instead", svcConf.MetricsPort, svcConf.Port)) w.WriteHeader(http.StatusOK) case "/readyz": + s.logger.Warn(fmt.Sprintf("call to deprecated /readyz endpoint on port %d, use port %d instead", svcConf.MetricsPort, svcConf.Port)) if s.readinessEnabled && svcConf.ReadinessProbe() { w.WriteHeader(http.StatusOK) } else { diff --git a/docs/other_resources/high_level_architecture.md b/docs/other_resources/high_level_architecture.md index d04be6643..3cfd356d8 100644 --- a/docs/other_resources/high_level_architecture.md +++ b/docs/other_resources/high_level_architecture.md @@ -35,13 +35,18 @@ process gets pushed to event subscribers. ## Readiness & Liveness probes +### HTTP + Flagd exposes HTTP liveness and readiness probes. These probes can be used for K8s deployments. -With default -start-up configurations, these probes are exposed at the following URLs, +With default start-up configurations, these probes are exposed on the service port (default: 8013) at the following URLs, + +- Liveness: +- Readiness: + +### gRPC -- Liveness: -- Readiness: +Flagd exposes a [standard gRPC health check](https://github.com/grpc/grpc/blob/master/doc/health-checking.md) on the service port (default: 8013). ### Definition of Liveness diff --git a/flagd/go.mod b/flagd/go.mod index eb8919f1f..5bb48a691 100644 --- a/flagd/go.mod +++ b/flagd/go.mod @@ -18,6 +18,8 @@ require ( buf.build/gen/go/open-feature/flagd/bufbuild/connect-go v1.9.0-20230720212818-3675556880a1.1 // indirect buf.build/gen/go/open-feature/flagd/grpc/go v1.3.0-20230710190440-2333a9579c1a.1 // indirect buf.build/gen/go/open-feature/flagd/protocolbuffers/go v1.31.0-20230720212818-3675556880a1.1 // indirect + connectrpc.com/connect v1.11.0 // indirect + connectrpc.com/grpchealth v1.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bufbuild/connect-go v1.10.0 // indirect github.com/bufbuild/connect-opentelemetry-go v0.4.0 // indirect diff --git a/flagd/go.sum b/flagd/go.sum index 2b22b4793..062a75ea2 100644 --- a/flagd/go.sum +++ b/flagd/go.sum @@ -404,6 +404,10 @@ cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1V cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +connectrpc.com/connect v1.11.0 h1:Av2KQXxSaX4vjqhf5Cl01SX4dqYADQ38eBtr84JSUBk= +connectrpc.com/connect v1.11.0/go.mod h1:3AGaO6RRGMx5IKFfqbe3hvK1NqLosFNP2BxDYTPmNPo= +connectrpc.com/grpchealth v1.2.0 h1:aHP33Bki+F2jPNI1mFVSFG7v0qJrgmfbg7X7nOdSj0M= +connectrpc.com/grpchealth v1.2.0/go.mod h1:fZos12C4p/ZaZC6OwBGZUM+i/fhnRhv75ax/6V/zIeM= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= From 4648c266f84a34b3431f686f5582dd839b0bd356 Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Thu, 24 Aug 2023 15:40:22 -0400 Subject: [PATCH 2/6] fixup: lint Signed-off-by: Todd Baert --- core/pkg/service/flag-evaluation/connect_service.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/pkg/service/flag-evaluation/connect_service.go b/core/pkg/service/flag-evaluation/connect_service.go index 5528fb227..82a1e9b29 100644 --- a/core/pkg/service/flag-evaluation/connect_service.go +++ b/core/pkg/service/flag-evaluation/connect_service.go @@ -220,10 +220,12 @@ func (s *ConnectService) startMetricsServer(svcConf service.Configuration) error s.metricsServer.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/healthz": - s.logger.Warn(fmt.Sprintf("call to deprecated /healthz endpoint on port %d, use port %d instead", svcConf.MetricsPort, svcConf.Port)) + s.logger.Warn( + fmt.Sprintf("/healthz endpoint on port %d is deprecated, use port %d instead", svcConf.MetricsPort, svcConf.Port)) w.WriteHeader(http.StatusOK) case "/readyz": - s.logger.Warn(fmt.Sprintf("call to deprecated /readyz endpoint on port %d, use port %d instead", svcConf.MetricsPort, svcConf.Port)) + s.logger.Warn( + fmt.Sprintf("/readyz endpoint on port %d is deprecated, use port %d instead", svcConf.MetricsPort, svcConf.Port)) if s.readinessEnabled && svcConf.ReadinessProbe() { w.WriteHeader(http.StatusOK) } else { From a00ae5a4cfb487177540f14d20d7453d314cb6fe Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Thu, 24 Aug 2023 15:47:12 -0400 Subject: [PATCH 3/6] fixup: minor refactor Signed-off-by: Todd Baert --- core/pkg/service/flag-evaluation/connect_service.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/pkg/service/flag-evaluation/connect_service.go b/core/pkg/service/flag-evaluation/connect_service.go index 82a1e9b29..0a68c2c5b 100644 --- a/core/pkg/service/flag-evaluation/connect_service.go +++ b/core/pkg/service/flag-evaluation/connect_service.go @@ -129,8 +129,7 @@ func (s *ConnectService) setupServer(svcConf service.Configuration) (net.Listene protojson.UnmarshalOptions{DiscardUnknown: true}, ) - path, handler := schemaConnectV1.NewServiceHandler(fes, append(svcConf.Options, marshalOpts)...) - mux.Handle(path, handler) + mux.Handle(schemaConnectV1.NewServiceHandler(fes, append(svcConf.Options, marshalOpts)...)) mux.Handle(grpchealth.NewHandler(grpchealth.NewStaticChecker( schemaConnectV1.ServiceName, ))) From 9c2fa3e09886b502aeb6e366cba765823ff9418c Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Fri, 25 Aug 2023 11:04:32 -0400 Subject: [PATCH 4/6] fixup: more compiler options Signed-off-by: Todd Baert --- .github/workflows/release-please.yaml | 2 +- flagd/build.Dockerfile | 2 +- flagd/profile.Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index e2a714d97..a526fd752 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -163,7 +163,7 @@ jobs: VERSION_NO_PREFIX: ${{ needs.release-please.outputs[format('{0}_version', matrix.path)] }} COMMIT: ${{ github.sha }} DATE: ${{ needs.release-please.outputs.date }} - BUILD_ARGS: '-a -ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}"' + BUILD_ARGS: '-a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}"' steps: - name: Checkout uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 diff --git a/flagd/build.Dockerfile b/flagd/build.Dockerfile index 1d887f340..e5ea6f329 100644 --- a/flagd/build.Dockerfile +++ b/flagd/build.Dockerfile @@ -29,7 +29,7 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=bind,source=./core,target=./core \ --mount=type=bind,source=./flagd,target=./flagd \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-build flagd/main.go + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-build flagd/main.go # # Use distroless as minimal base image to package the manager binary # # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/flagd/profile.Dockerfile b/flagd/profile.Dockerfile index 77ac3f7bf..28b57a21e 100644 --- a/flagd/profile.Dockerfile +++ b/flagd/profile.Dockerfile @@ -29,7 +29,7 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=bind,source=./core,target=./core \ --mount=type=bind,source=./flagd,target=./flagd \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-build ./flagd/main.go ./flagd/profiler.go + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-build ./flagd/main.go ./flagd/profiler.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details From c26e569212ccdfd7a24350994470dc8abaaca775 Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Fri, 25 Aug 2023 11:10:27 -0400 Subject: [PATCH 5/6] fixup: conflictPolicy=ignore in prod Signed-off-by: Todd Baert --- .github/workflows/release-please.yaml | 2 +- flagd/build.Dockerfile | 2 +- flagd/profile.Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index a526fd752..56fd4e06f 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -163,7 +163,7 @@ jobs: VERSION_NO_PREFIX: ${{ needs.release-please.outputs[format('{0}_version', matrix.path)] }} COMMIT: ${{ github.sha }} DATE: ${{ needs.release-please.outputs.date }} - BUILD_ARGS: '-a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}"' + BUILD_ARGS: '-a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=ignore -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}"' steps: - name: Checkout uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 diff --git a/flagd/build.Dockerfile b/flagd/build.Dockerfile index e5ea6f329..221807f64 100644 --- a/flagd/build.Dockerfile +++ b/flagd/build.Dockerfile @@ -29,7 +29,7 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=bind,source=./core,target=./core \ --mount=type=bind,source=./flagd,target=./flagd \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-build flagd/main.go + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=ignore -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-build flagd/main.go # # Use distroless as minimal base image to package the manager binary # # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/flagd/profile.Dockerfile b/flagd/profile.Dockerfile index 28b57a21e..eab9565fa 100644 --- a/flagd/profile.Dockerfile +++ b/flagd/profile.Dockerfile @@ -29,7 +29,7 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=bind,source=./core,target=./core \ --mount=type=bind,source=./flagd,target=./flagd \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-build ./flagd/main.go ./flagd/profiler.go + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=ignore -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-build ./flagd/main.go ./flagd/profiler.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details From 465add4adef0d072d984a32750f8e061eb3a5fac Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Fri, 25 Aug 2023 12:45:00 -0400 Subject: [PATCH 6/6] fixup: more doc Signed-off-by: Todd Baert --- docs/other_resources/high_level_architecture.md | 2 +- flagd-proxy/build.Dockerfile | 2 +- web-docs/concepts/architecture.md | 13 +++++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/other_resources/high_level_architecture.md b/docs/other_resources/high_level_architecture.md index 3cfd356d8..1e18f16a5 100644 --- a/docs/other_resources/high_level_architecture.md +++ b/docs/other_resources/high_level_architecture.md @@ -46,7 +46,7 @@ With default start-up configurations, these probes are exposed on the service po ### gRPC -Flagd exposes a [standard gRPC health check](https://github.com/grpc/grpc/blob/master/doc/health-checking.md) on the service port (default: 8013). +Flagd exposes a [standard gRPC liveness check](https://github.com/grpc/grpc/blob/master/doc/health-checking.md) on the service port (default: 8013). ### Definition of Liveness diff --git a/flagd-proxy/build.Dockerfile b/flagd-proxy/build.Dockerfile index 2e2273a6e..e18bbdb89 100644 --- a/flagd-proxy/build.Dockerfile +++ b/flagd-proxy/build.Dockerfile @@ -32,7 +32,7 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \ --mount=type=bind,source=./core,target=./core \ --mount=type=bind,source=./flagd,target=./flagd \ --mount=type=bind,source=./flagd-proxy,target=./flagd-proxy \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-proxy flagd-proxy/main.go + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=ignore -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o /bin/flagd-proxy flagd-proxy/main.go # # Use distroless as minimal base image to package the manager binary # # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/web-docs/concepts/architecture.md b/web-docs/concepts/architecture.md index 08a4d910e..294187d45 100644 --- a/web-docs/concepts/architecture.md +++ b/web-docs/concepts/architecture.md @@ -33,13 +33,18 @@ process gets pushed to event subscribers. ## Readiness & Liveness probes +### HTTP + Flagd exposes HTTP liveness and readiness probes. These probes can be used for K8s deployments. -With default -start-up configurations, these probes are exposed at the following URLs, +With default start-up configurations, these probes are exposed on the service port (default: 8013) at the following URLs, + +- Liveness: +- Readiness: + +### gRPC -- Liveness: -- Readiness: +Flagd exposes a [standard gRPC liveness check](https://github.com/grpc/grpc/blob/master/doc/health-checking.md) on the service port (default: 8013). ### Definition of Liveness