From 772bf9a22e4826ffaa2a6aaed2b6cecfc9fd9906 Mon Sep 17 00:00:00 2001 From: Jonathan Hess Date: Thu, 14 Sep 2023 12:58:02 -0600 Subject: [PATCH] docs: Update the example healthcheck config to match the default operator configuration. chore: code review comments. --- examples/k8s-health-check/README.md | 57 +++--- .../proxy_with_http_health_check.yaml | 184 +++++++++++------- 2 files changed, 144 insertions(+), 97 deletions(-) diff --git a/examples/k8s-health-check/README.md b/examples/k8s-health-check/README.md index e16b2aad8..01c12b311 100644 --- a/examples/k8s-health-check/README.md +++ b/examples/k8s-health-check/README.md @@ -54,7 +54,7 @@ livenessProbe: # We recommend adding a liveness probe to the proxy sidecar container. httpGet: path: /liveness - port: 9090 + port: 9801 # Number of seconds after the container has started before the first probe is scheduled. Defaults to 0. # Not necessary when the startup probe is in use. initialDelaySeconds: 0 @@ -74,35 +74,40 @@ livenessProbe: # We do not recommend adding a readiness probe under most circumstances ``` -2. Add `--http-address` and `--http-port` (optional) to your +2. Enable the health checks by setting `--http-address` and `--http-port` (optional) to your proxy container configuration under `command: `. > [proxy_with_http_health_check.yaml](proxy_with_http_health_check.yaml#L53-L76) ```yaml args: - # If connecting from a VPC-native GKE cluster, you can use the - # following flag to have the proxy connect over private IP - # - "--private-ip" - - # Enable HTTP health checks - - "--health-check" - - # Listen on all addresses so the kubelet can reach the endpoints - - "--http-address=0.0.0.0" - - # Set the port where the HTTP server listens - # - "--http-port=9090" - - # Enable structured logging with LogEntry format: - - "--structured-logs" - - # This flag specifies where the service account key can be found - # Remove this argument if you are using workload identity - - "--credentials-file=/secrets/service_account.json" +# Replace with the instance connection +# name in the format: "project_name:region:instance_name" +# Replace with the port that the proxy should open +# to listen for database connections from the application +- ?port= + +env: +# It can be easier to manage the k8s configuration file when you +# use environment variables instead of CLI flags. This is the +# recommended configuration. This configuration is enabled by default +# when the cloud-sql-proxy-operator configures a proxy image + +# Enable HTTP healthchecks on port 9801. This enables /liveness, +# /readiness and /startup health check endpoints. Allow connections +# listen for connections on any interface (0.0.0.0) so that the +# k8s management components can reach these endpoints. +- name: CSQL_PROXY_HEALTH_CHECK + value: "true" +- name: CSQL_PROXY_HTTP_PORT + value: "9801" +- name: CSQL_PROXY_HTTP_ADDRESS + value: 0.0.0.0 + +# Configure the proxy to exit gracefully when sent a k8s configuration +# file. +- name: CSQL_PROXY_EXIT_ZERO_ON_SIGTERM + value: "true" - # Replace DB_PORT with the port the proxy should listen on - - "--port=" - - "" ``` ### Readiness Health Check Configuration @@ -145,7 +150,7 @@ seconds, 6 x 30sec = 3 minutes.) readinessProbe: httpGet: path: /readiness - port: 9090 + port: 9801 initialDelaySeconds: 30 # 30 sec period x 6 failures = 3 min until the pod is terminated periodSeconds: 30 @@ -176,7 +181,7 @@ for more than 1 minute. readinessProbe: httpGet: path: /readiness - port: 9090 + port: 9801 initialDelaySeconds: 10 # 5 sec period x 12 failures = 60 sec until the pod is terminated periodSeconds: 5 diff --git a/examples/k8s-health-check/proxy_with_http_health_check.yaml b/examples/k8s-health-check/proxy_with_http_health_check.yaml index ad3534304..cb82c5ec4 100644 --- a/examples/k8s-health-check/proxy_with_http_health_check.yaml +++ b/examples/k8s-health-check/proxy_with_http_health_check.yaml @@ -49,90 +49,99 @@ spec: - name: cloud-sql-proxy # It is recommended to use the latest version of the Cloud SQL Auth Proxy # Make sure to update on a regular schedule! - image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.1.0 + image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.6.0 + imagePullPolicy: IfNotPresent + args: - # If connecting from a VPC-native GKE cluster, you can use the - # following flag to have the proxy connect over private IP - # - "--private-ip" + # Replace with the instance connection + # name in the format: "project_name:region:instance_name" + # Replace with the port that the proxy should open + # to listen for database connections from the application + - ?port= - # Enable HTTP health checks - - "--health-check" + env: + # Using environment variables instead of CLI arguments to configure the + # proxy k8s configuration can make it easier to read your + # k8s configuration files. + # + # This is the recommended configuration for your proxy pod template. + # It is used by the cloud-sql-proxy-operator when configuring the + # proxy container. - # Listen on all addresses so the kubelet can reach the endpoints - - "--http-address=0.0.0.0" + # If connecting from a VPC-native GKE cluster, you can use the + # following flag to have the proxy connect over private IP + # - name: CSQL_PROXY_PRIVATE_IP + # value: "true" - # Set the port where the HTTP server listens - # - "--http-port=9090" + # Enable HTTP healthchecks on port 9801. This enables /liveness, + # /readiness and /startup health check endpoints. Allow connections + # listen for connections on any interface (0.0.0.0) so that the + # k8s management components can reach these endpoints. + - name: CSQL_PROXY_HEALTH_CHECK + value: "true" + - name: CSQL_PROXY_HTTP_PORT + value: "9801" + - name: CSQL_PROXY_HTTP_ADDRESS + value: 0.0.0.0 - # Enable structured logging with LogEntry format: - - "--structured-logs" + # Configure the proxy to exit gracefully when sent a k8s configuration + # file. + - name: CSQL_PROXY_EXIT_ZERO_ON_SIGTERM + value: "true" - # This flag specifies where the service account key can be found - # Remove this argument if you are using workload identity - - "--credentials-file=/secrets/service_account.json" + # Enable the admin api server (which only listens for local connections) + # and enable the /quitquitquit endpoint. This allows other pods + # to shut down the proxy gracefully when they are ready to exit. + - name: CSQL_PROXY_QUITQUITQUIT + value: "true" + - name: CSQL_PROXY_ADMIN_PORT + value: "9092" + - name: CSQL_PROXY_USER_AGENT + value: cloud-sql-proxy-operator/unknown - # Replace DB_PORT with the port the proxy should listen on - - "--port=" - - "" + # Enable structured logging with LogEntry format + - name: CSQL_PROXY_STRUCTURED_LOGS + value: "true" - securityContext: - # The default Cloud SQL Auth Proxy image runs as the - # "nonroot" user and group (uid: 65532) by default. - runAsNonRoot: true - # Use a read-only filesystem - readOnlyRootFilesystem: true - # Do not allow privilege escalation - allowPrivilegeEscalation : false - volumeMounts: - - name: - mountPath: /secrets/ - readOnly: true - # Resource configuration depends on an application's requirements. You - # should adjust the following values based on what your application - # needs. For details, see https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - resources: - requests: - # The proxy's memory use scales linearly with the number of active - # connections. Fewer open connections will use less memory. Adjust - # this value based on your application's requirements. - memory: "2Gi" - # The proxy's CPU use scales linearly with the amount of IO between - # the database and the application. Adjust this value based on your - # application's requirements. - cpu: "1" - # Recommended configurations for health check probes. - # Probe parameters can be adjusted to best fit the requirements of your application. - # For details, see https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + # Configure kubernetes to call the /quitquitquit endpoint on the + # admin server before sending SIGTERM to the proxy before stopping + # the pod. This will give the proxy more time to gracefully exit. + lifecycle: + preStop: + httpGet: + path: /quitquitquit + port: 9092 + scheme: HTTP + + # The /startup probe returns OK when the proxy is ready to receive + # connections from the application. In this example, k8s will check + # once a second for 60 seconds. + # + # We strongly recommend adding a startup probe to the proxy sidecar + # container. This will ensure that service traffic will be routed to + # the pod only after the proxy has successfully started. startupProbe: - # The /startup probe returns OK when the proxy is ready to receive - # connections from the application. In this example, k8s will check - # once a second for 20 seconds. - # - # We strongly recommend adding a startup probe to the proxy sidecar - # container. This will ensure that service traffic will be routed to - # the pod only after the proxy has successfully started. + failureThreshold: 60 httpGet: path: /startup - port: 9090 + port: 9801 + scheme: HTTP periodSeconds: 1 - timeoutSeconds: 5 - failureThreshold: 20 + successThreshold: 1 + timeoutSeconds: 10 + # The /liveness probe returns OK as soon as the proxy application has + # begun its startup process and continues to return OK until the + # process stops. + # + # We recommend adding a liveness probe to the proxy sidecar container. livenessProbe: - # The /liveness probe returns OK as soon as the proxy application has - # begun its startup process and continues to return OK until the - # process stops. - # - # We recommend adding a liveness probe to the proxy sidecar container. + failureThreshold: 3 httpGet: path: /liveness - port: 9090 - # Number of seconds after the container has started before the first probe is scheduled. Defaults to 0. - # Not necessary when the startup probe is in use. - initialDelaySeconds: 0 - # Frequency of the probe. - periodSeconds: 60 - # Number of seconds after which the probe times out. - timeoutSeconds: 30 + port: 9801 + scheme: HTTP + # The probe will be checked every 10 seconds. + periodSeconds: 10 # Number of times the probe is allowed to fail before the transition # from healthy to failure state. # @@ -141,7 +150,9 @@ spec: # before its expiration. If those five minutes lapse without a # successful refresh, the liveness probe will fail and the pod will be # restarted. - failureThreshold: 5 + successThreshold: 1 + # The probe will fail if it does not respond in 10 seconds + timeoutSeconds: 10 readinessProbe: # The /readiness probe returns OK when the proxy can establish # a new connections to its databases. @@ -151,7 +162,7 @@ spec: # interruption to the application. See README.md for more detail. httpGet: path: /readiness - port: 9090 + port: 9801 initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 10 @@ -159,6 +170,37 @@ spec: # Defaults to 1 for readiness probe. successThreshold: 1 failureThreshold: 6 + + # Declare the HTTP Port so that k8s components can reach the + # metrics and health check endpoints. + ports: + - containerPort: 9801 + protocol: TCP + # You should use resource requests/limits as a best practice to prevent + # pods from consuming too many resources and affecting the execution of + # other pods. You should adjust the following values based on what your + # application needs. For details, see + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + requests: + # The proxy's memory use scales linearly with the number of active + # connections. Fewer open connections will use less memory. Adjust + # this value based on your application's requirements. + memory: "2Gi" + # The proxy's CPU use scales linearly with the amount of IO between + # the database and the application. Adjust this value based on your + # application's requirements. + cpu: "1" + securityContext: + # The default Cloud SQL Auth Proxy image runs as the + # "nonroot" user and group (uid: 65532) by default. + runAsNonRoot: true + # Use a read-only filesystem + readOnlyRootFilesystem: true + # Do not allow privilege escalation + allowPrivilegeEscalation : false + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File volumes: - name: secret: