Skip to content

Commit

Permalink
Mutating Webhook - Part 1 (#1560)
Browse files Browse the repository at this point in the history 
* Implement basic webhook in instrumentor
* Modify Helm and CLI to install new mutating webhook
* Work with and without `cert-manager`

---------

Co-authored-by: Amir Blum <amirgiraffe@gmail.com>
  • Loading branch information
@edeNFed @blumamir
edeNFed and blumamir authored Oct 1, 2024
1 parent 4ae11ce commit 217e289
Show file tree
Hide file tree
Showing 14 changed files with 821 additions and 44 deletions.
262 changes: 257 additions & 5 deletions cli/cmd/resources/instrumentor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,18 @@ package resources

import (
"context"
"fmt"

"github.com/odigos-io/odigos/cli/cmd/resources/resourcemanager"
"github.com/odigos-io/odigos/cli/pkg/containers"
"github.com/odigos-io/odigos/cli/pkg/crypto"
"github.com/odigos-io/odigos/cli/pkg/kube"
"github.com/odigos-io/odigos/common"
"sigs.k8s.io/controller-runtime/pkg/client"

certv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
Expand All @@ -18,10 +23,12 @@ import (
)

const (
InstrumentorServiceName = "instrumentor"
InstrumentorDeploymentName = "odigos-instrumentor"
InstrumentorAppLabelValue = "odigos-instrumentor"
InstrumentorContainerName = "manager"
InstrumentorServiceName = "instrumentor"
InstrumentorDeploymentName = "odigos-instrumentor"
InstrumentorAppLabelValue = "odigos-instrumentor"
InstrumentorContainerName = "manager"
InstrumentorWebhookSecretName = "instrumentor-webhook-cert"
InstrumentorWebhookVolumeName = "webhook-cert"
)

func NewInstrumentorServiceAccount(ns string) *corev1.ServiceAccount {
Expand Down Expand Up @@ -219,6 +226,193 @@ func NewInstrumentorClusterRoleBinding(ns string) *rbacv1.ClusterRoleBinding {
}
}

func isCertManagerInstalled(ctx context.Context, c *kube.Client) bool {
// Check if CRD is installed
_, err := c.ApiExtensions.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "issuers.cert-manager.io", metav1.GetOptions{})
if err != nil {
return false
}

return true
}

func NewInstrumentorIssuer(ns string) *certv1.Issuer {
return &certv1.Issuer{
TypeMeta: metav1.TypeMeta{
Kind: "Issuer",
APIVersion: "cert-manager.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "selfsigned-issuer",
Namespace: ns,
Labels: map[string]string{
"app.kubernetes.io/name": "issuer",
"app.kubernetes.io/instance": "selfsigned-issuer",
"app.kubernetes.io/component": "certificate",
"app.kubernetes.io/created-by": "instrumentor",
"app.kubernetes.io/part-of": "odigos",
},
},
Spec: certv1.IssuerSpec{
IssuerConfig: certv1.IssuerConfig{
SelfSigned: &certv1.SelfSignedIssuer{},
},
},
}
}

func NewInstrumentorCertificate(ns string) *certv1.Certificate {
return &certv1.Certificate{
TypeMeta: metav1.TypeMeta{
Kind: "Certificate",
APIVersion: "cert-manager.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "serving-cert",
Namespace: ns,
Labels: map[string]string{
"app.kubernetes.io/name": "instrumentor-cert",
"app.kubernetes.io/instance": "instrumentor-cert",
"app.kubernetes.io/component": "certificate",
"app.kubernetes.io/created-by": "instrumentor",
"app.kubernetes.io/part-of": "odigos",
},
},
Spec: certv1.CertificateSpec{
DNSNames: []string{
fmt.Sprintf("odigos-instrumentor.%s.svc", ns),
fmt.Sprintf("odigos-instrumentor.%s.svc.cluster.local", ns),
},
IssuerRef: cmmeta.ObjectReference{
Kind: "Issuer",
Name: "selfsigned-issuer",
},
SecretName: InstrumentorWebhookSecretName,
},
}
}

func NewInstrumentorService(ns string) *corev1.Service {
return &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "odigos-instrumentor",
Namespace: ns,
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "webhook-server",
Port: 9443,
TargetPort: intstr.FromInt(9443),
},
},
Selector: map[string]string{
"app.kubernetes.io/name": InstrumentorAppLabelValue,
},
},
}
}

func NewMutatingWebhookConfiguration(ns string, caBundle []byte) *admissionregistrationv1.MutatingWebhookConfiguration {
webhook := &admissionregistrationv1.MutatingWebhookConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "MutatingWebhookConfiguration",
APIVersion: "admissionregistration.k8s.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "mutating-webhook-configuration",
Labels: map[string]string{
"app.kubernetes.io/name": "pod-mutating-webhook",
"app.kubernetes.io/instance": "mutating-webhook-configuration",
"app.kubernetes.io/component": "webhook",
"app.kubernetes.io/created-by": "instrumentor",
"app.kubernetes.io/part-of": "odigos",
},
},
Webhooks: []admissionregistrationv1.MutatingWebhook{
{
Name: "pod-mutating-webhook.odigos.io",
ClientConfig: admissionregistrationv1.WebhookClientConfig{
Service: &admissionregistrationv1.ServiceReference{
Name: "odigos-instrumentor",
Namespace: ns,
Path: ptrString("/mutate--v1-pod"),
Port: intPtr(9443),
},
},
Rules: []admissionregistrationv1.RuleWithOperations{
{
Operations: []admissionregistrationv1.OperationType{
admissionregistrationv1.Create,
admissionregistrationv1.Update,
},
Rule: admissionregistrationv1.Rule{
APIGroups: []string{""},
APIVersions: []string{"v1"},
Resources: []string{"pods"},
Scope: ptrGeneric(admissionregistrationv1.NamespacedScope),
},
},
},
FailurePolicy: ptrGeneric(admissionregistrationv1.Ignore),
ReinvocationPolicy: ptrGeneric(admissionregistrationv1.IfNeededReinvocationPolicy),
SideEffects: ptrGeneric(admissionregistrationv1.SideEffectClassNone),
TimeoutSeconds: intPtr(10),
ObjectSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"odigos.io/inject-instrumentation": "true",
},
},
AdmissionReviewVersions: []string{
"v1",
},
},
},
}

if caBundle == nil {
webhook.Annotations = map[string]string{
"cert-manager.io/inject-ca-from": fmt.Sprintf("%s/serving-cert", ns),
}
} else {
webhook.Webhooks[0].ClientConfig.CABundle = caBundle
}

return webhook
}

func NewInstrumentorTLSSecret(ns string, cert *crypto.Certificate) *corev1.Secret {
return &corev1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: InstrumentorWebhookSecretName,
Namespace: ns,
Labels: map[string]string{
"app.kubernetes.io/name": "instrumentor-cert",
"app.kubernetes.io/instance": "instrumentor-cert",
"app.kubernetes.io/component": "certificate",
"app.kubernetes.io/created-by": "instrumentor",
"app.kubernetes.io/part-of": "odigos",
},
Annotations: map[string]string{
"helm.sh/hook": "pre-install,pre-upgrade",
"helm.sh/hook-delete-policy": "before-hook-creation",
},
},
Data: map[string][]byte{
"tls.crt": []byte(cert.Cert),
"tls.key": []byte(cert.Key),
},
}
}

func NewInstrumentorDeployment(ns string, version string, telemetryEnabled bool, imagePrefix string, imageName string) *appsv1.Deployment {
args := []string{
"--health-probe-bind-address=:8081",
Expand All @@ -230,7 +424,7 @@ func NewInstrumentorDeployment(ns string, version string, telemetryEnabled bool,
args = append(args, "--telemetry-disabled")
}

return &appsv1.Deployment{
dep := &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
Expand Down Expand Up @@ -290,6 +484,20 @@ func NewInstrumentorDeployment(ns string, version string, telemetryEnabled bool,
},
},
},
Ports: []corev1.ContainerPort{
{
Name: "webhook-server",
ContainerPort: 9443,
Protocol: corev1.ProtocolTCP,
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: InstrumentorWebhookVolumeName,
ReadOnly: true,
MountPath: "/tmp/k8s-webhook-server/serving-certs",
},
},
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"cpu": resource.MustParse("500m"),
Expand Down Expand Up @@ -324,12 +532,25 @@ func NewInstrumentorDeployment(ns string, version string, telemetryEnabled bool,
SecurityContext: &corev1.PodSecurityContext{
RunAsNonRoot: ptrbool(true),
},
Volumes: []corev1.Volume{
{
Name: InstrumentorWebhookVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: InstrumentorWebhookSecretName,
DefaultMode: ptrint32(420),
},
},
},
},
},
},
Strategy: appsv1.DeploymentStrategy{},
MinReadySeconds: 0,
},
}

return dep
}

func ptrint32(i int32) *int32 {
Expand Down Expand Up @@ -363,12 +584,43 @@ func NewInstrumentorResourceManager(client *kube.Client, ns string, config *comm
func (a *instrumentorResourceManager) Name() string { return "Instrumentor" }

func (a *instrumentorResourceManager) InstallFromScratch(ctx context.Context) error {
certManagerInstalled := isCertManagerInstalled(ctx, a.client)
resources := []client.Object{
NewInstrumentorServiceAccount(a.ns),
NewInstrumentorRoleBinding(a.ns),
NewInstrumentorClusterRole(),
NewInstrumentorClusterRoleBinding(a.ns),
NewInstrumentorDeployment(a.ns, a.odigosVersion, a.config.TelemetryEnabled, a.config.ImagePrefix, a.config.InstrumentorImage),
NewInstrumentorService(a.ns),
}

if certManagerInstalled {
resources = append([]client.Object{NewInstrumentorIssuer(a.ns),
NewInstrumentorCertificate(a.ns),
NewMutatingWebhookConfiguration(a.ns, nil),
},
resources...)
} else {
ca, err := crypto.GenCA("odigos-instrumentor", 365)
if err != nil {
return fmt.Errorf("failed to generate CA: %w", err)
}

altNames := []string{
fmt.Sprintf("odigos-instrumentor.%s.svc", a.ns),
fmt.Sprintf("odigos-instrumentor.%s.svc.cluster.local", a.ns),
}

cert, err := crypto.GenerateSignedCertificate("serving-cert", nil, altNames, 365, ca)
if err != nil {
return fmt.Errorf("failed to generate signed certificate: %w", err)
}

resources = append([]client.Object{NewInstrumentorTLSSecret(a.ns, &cert),
NewMutatingWebhookConfiguration(a.ns, []byte(cert.Cert)),
},
resources...)
}

return a.client.ApplyResources(ctx, a.config.ConfigVersion, resources)
}
8 changes: 8 additions & 0 deletions cli/cmd/resources/owntelemetry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,14 @@ func int64Ptr(n int64) *int64 {
return &n
}

func ptrString(s string) *string {
return &s
}

func ptrGeneric[T any](v T) *T {
return &v
}

type ownTelemetryResourceManager struct {
client *kube.Client
ns string
Expand Down
Loading

0 comments on commit 217e289

Please sign in to comment.