Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding support for lifecycle hooks and health probe for sidecars #1482

Closed
wants to merge 7 commits into from
Closed

Adding support for lifecycle hooks and health probe for sidecars #1482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4b64e86
configuring probes for envoy sidecars
narendrapatel Sep 6, 2022
46d6266
Adding support for lifecycle hooks and health probe
narendrapatel Sep 9, 2022
68d7cbe
resolving conflicts
narendrapatel Sep 9, 2022
e543f22
adding graceful start up and shut down for sidecar
narendrapatel Dec 5, 2022
20a1351
fixing linter issues
narendrapatel Dec 5, 2022
e8fc978
fix linter issues
narendrapatel Dec 6, 2022
c22b393
updating patch to be compatible with versions > 1.0.0
narendrapatel Dec 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions control-plane/connect-inject/constants/annotations_and_labels.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ const (
AnnotationSidecarProxyMemoryLimit = "consul.hashicorp.com/sidecar-proxy-memory-limit"
AnnotationSidecarProxyMemoryRequest = "consul.hashicorp.com/sidecar-proxy-memory-request"

// annotation makes sidecar shutdown gracefully.
AnnotationSidecarProxyGracefulShutdown = "consul.hashicorp.com/sidecar-proxy-graceful-shutdown"

// annotation to hold starting of app containers till sidecar proxy is started.
AnnotationSidecarProxyHoldApplicationUntilProxyStarts = "consul.hashicorp.com/sidecar-hold-app-until-proxy-starts"

// annotations for sidecar volumes.
AnnotationConsulSidecarUserVolume = "consul.hashicorp.com/consul-sidecar-user-volume"
AnnotationConsulSidecarUserVolumeMount = "consul.hashicorp.com/consul-sidecar-user-volume-mount"
Expand Down
62 changes: 62 additions & 0 deletions control-plane/connect-inject/webhook/consul_dataplane_sidecar.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,20 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
container.VolumeMounts = append(container.VolumeMounts, volumeMounts...)
}

var lifecycle corev1.Lifecycle

preStop, err := w.envoySidecarGracefulShutdown(pod)
if err == nil && preStop != nil {
lifecycle.PreStop = preStop
}

postStart, err := w.envoySidecarHoldApplicationUntilProxyStarts(pod)
if err == nil && postStart != nil {
lifecycle.PostStart = postStart
}

container.Lifecycle = &lifecycle

tproxyEnabled, err := common.TransparentProxyEnabled(namespace, pod, w.EnableTransparentProxy)
if err != nil {
return corev1.Container{}, err
Expand Down Expand Up @@ -140,6 +154,54 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
return container, nil
}

// Configures graceful shut down for the sidecar.
func (w *MeshWebhook) envoySidecarGracefulShutdown(pod corev1.Pod) (*corev1.Handler, error) {

grace, err := strconv.ParseBool(pod.Annotations[constants.AnnotationSidecarProxyGracefulShutdown])

if err != nil || !grace {
return nil, err
}

preStop := &corev1.Handler{
Exec: &corev1.ExecAction{
Command: []string{
"/bin/sh",
"-c",
"while [ $(netstat -plunt | grep tcp | grep -v envoy | grep -v consul-dataplane | wc -l) -ne 0 ]; do sleep 1; done",
},
},
}

return preStop, nil
}

// Ensures that the sidecar is the first container to start up.
func (w *MeshWebhook) envoySidecarHoldApplicationUntilProxyStarts(pod corev1.Pod) (*corev1.Handler, error) {

hold, err := strconv.ParseBool(pod.Annotations[constants.AnnotationSidecarProxyHoldApplicationUntilProxyStarts])

if err != nil || !hold {
return nil, err
}
postStart := &corev1.Handler{
Exec: &corev1.ExecAction{
Command: []string{
"/bin/sh",
"-c",
`total_time=0; until wget --spider localhost:19000;` +
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The /ready endpoint should be more appropriate for this check IMHO:

#!/bin/sh

while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:19000/ready)" != "200" ]]; do
  sleep 1
done

Copy link

@komapa komapa Aug 10, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We do while [[ "$(curl -sf http://127.0.0.1:19000/ready)" != "LIVE" ]]; do sleep 1; done with great success :)

`do echo Waiting for Sidecar;` +
`sleep 3; total_time=$(($total_time + 3)); echo $total_time;` +
`if [ $total_time -gt 120 ]; then echo Sidecar not running, timeout reached. Exiting....; exit 1; fi; done;` +
`echo Sidecar available`,
},
},
}

return postStart, err

}

func (w *MeshWebhook) getContainerSidecarArgs(namespace corev1.Namespace, mpi multiPortInfo, bearerTokenFile string, pod corev1.Pod) ([]string, error) {
proxyIDFileName := "/consul/connect-inject/proxyid"
if mpi.serviceName != "" {
Expand Down
12 changes: 10 additions & 2 deletions control-plane/connect-inject/webhook/mesh_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi
w.Log.Error(err, "error configuring injection sidecar container", "request name", req.Name)
return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring injection sidecar container: %s", err))
}
pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar)
pod.Spec.Containers = injectSidecar(pod, pod.Spec.Containers, envoySidecar)
} else {
// For multi port pods, check for unsupported cases, mount all relevant service account tokens, and mount an init
// container and envoy sidecar per port. Tproxy, metrics, and metrics merging are not supported for multi port pods.
Expand Down Expand Up @@ -373,7 +373,7 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi
w.Log.Error(err, "error configuring injection sidecar container", "request name", req.Name)
return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring injection sidecar container: %s", err))
}
pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar)
pod.Spec.Containers = injectSidecar(pod, pod.Spec.Containers, envoySidecar)
}
}

Expand Down Expand Up @@ -480,6 +480,14 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi
return admission.Patched(fmt.Sprintf("valid %s request", pod.Kind), patches...)
}

func injectSidecar(pod corev1.Pod, containers []corev1.Container, sidecar corev1.Container) []corev1.Container {
hold, err := strconv.ParseBool(pod.Annotations[constants.AnnotationSidecarProxyHoldApplicationUntilProxyStarts])
if err != nil || !hold {
return append(containers, sidecar)
}
return append([]corev1.Container{sidecar}, containers...)
}

// overwriteProbes overwrites readiness/liveness probes of this pod when
// both transparent proxy is enabled and overwrite probes is true for the pod.
func (w *MeshWebhook) overwriteProbes(ns corev1.Namespace, pod *corev1.Pod) error {
Expand Down